2023-11-19 01:29:50,865 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed 2023-11-19 01:29:50,880 main DEBUG Took 0.011946 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-19 01:29:50,880 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-19 01:29:50,880 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-19 01:29:50,881 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-19 01:29:50,883 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,892 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-19 01:29:50,908 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,910 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,911 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,912 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,913 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,914 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,915 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,916 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,917 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,918 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,918 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,919 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:29:50,920 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,921 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-19 01:29:50,923 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:29:50,925 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-19 01:29:50,927 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-19 01:29:50,927 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-19 01:29:50,929 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-19 01:29:50,929 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-19 01:29:50,944 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-19 01:29:50,947 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-19 01:29:50,952 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-19 01:29:50,952 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-19 01:29:50,952 main DEBUG createAppenders(={Console}) 2023-11-19 01:29:50,954 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed initialized 2023-11-19 01:29:50,954 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed 2023-11-19 01:29:50,954 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed OK. 2023-11-19 01:29:50,955 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-19 01:29:50,955 main DEBUG OutputStream closed 2023-11-19 01:29:50,955 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-19 01:29:50,956 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-19 01:29:50,956 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@498d318c OK 2023-11-19 01:29:50,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-19 01:29:51,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-19 01:29:51,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-19 01:29:51,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-19 01:29:51,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-19 01:29:51,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-19 01:29:51,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-19 01:29:51,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-19 01:29:51,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-19 01:29:51,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-19 01:29:51,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-19 01:29:51,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-19 01:29:51,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-19 01:29:51,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-19 01:29:51,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-19 01:29:51,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-19 01:29:51,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-19 01:29:51,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-19 01:29:51,010 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19 01:29:51,011 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4d5b6aac) with optional ClassLoader: null 2023-11-19 01:29:51,011 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-19 01:29:51,012 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@4d5b6aac] started OK. 2023-11-19T01:29:51,043 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestRemoteRestore timeout: 13 mins 2023-11-19 01:29:51,048 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-19 01:29:51,048 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19T01:29:51,352 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06 2023-11-19T01:29:51,357 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:29:51,360 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:51,361 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-19T01:29:51,365 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7, deleteOnExit=true 2023-11-19T01:29:51,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-19T01:29:51,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/test.cache.data in system properties and HBase conf 2023-11-19T01:29:51,366 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:29:51,366 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:29:51,367 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:29:51,367 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:29:51,367 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:29:51,476 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-19T01:29:51,957 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-19T01:29:51,962 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:29:51,962 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:29:51,963 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:29:51,963 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:29:51,963 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:29:51,964 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:29:51,964 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:29:51,964 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:29:51,965 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:29:51,965 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:29:51,965 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:29:51,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:29:51,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:29:51,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-19T01:29:52,462 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:29:52,467 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:29:52,770 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-19T01:29:52,904 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@7f6e0dfb via org.mortbay.log.Slf4jLog 2023-11-19T01:29:52,919 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:29:52,962 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:29:52,999 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir/Jetty_localhost_42801_hdfs____.5pz9ln/webapp 2023-11-19T01:29:53,138 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:42801 2023-11-19T01:29:53,163 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:29:53,164 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:29:53,723 WARN [Listener at localhost/38223 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:29:53,796 WARN [Listener at localhost/38223 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:29:53,816 WARN [Listener at localhost/38223 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:29:53,823 INFO [Listener at localhost/38223 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:29:53,832 INFO [Listener at localhost/38223 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir/Jetty_localhost_33255_datanode____k98uzd/webapp 2023-11-19T01:29:53,949 INFO [Listener at localhost/38223 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33255 2023-11-19T01:29:54,240 WARN [Listener at localhost/42919 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:29:54,541 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:29:54,710 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb1b4eedcf6a27951: Processing first storage report for DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba from datanode bee95ccc-1747-497c-9846-95b6ca0b696a 2023-11-19T01:29:54,711 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb1b4eedcf6a27951: from storage DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba node DatanodeRegistration(127.0.0.1:34535, datanodeUuid=bee95ccc-1747-497c-9846-95b6ca0b696a, infoPort=45535, infoSecurePort=0, ipcPort=42919, storageInfo=lv=-57;cid=testClusterID;nsid=293027700;c=1700357392544), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-11-19T01:29:54,712 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb1b4eedcf6a27951: Processing first storage report for DS-f57fece8-7b03-433c-8917-eedfe681b85d from datanode bee95ccc-1747-497c-9846-95b6ca0b696a 2023-11-19T01:29:54,712 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb1b4eedcf6a27951: from storage DS-f57fece8-7b03-433c-8917-eedfe681b85d node DatanodeRegistration(127.0.0.1:34535, datanodeUuid=bee95ccc-1747-497c-9846-95b6ca0b696a, infoPort=45535, infoSecurePort=0, ipcPort=42919, storageInfo=lv=-57;cid=testClusterID;nsid=293027700;c=1700357392544), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:29:54,800 DEBUG [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06 2023-11-19T01:29:54,941 INFO [Listener at localhost/42919 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/zookeeper_0, clientPort=53145, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-19T01:29:54,961 INFO [Listener at localhost/42919 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53145 2023-11-19T01:29:54,968 INFO [Listener at localhost/42919 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:54,970 INFO [Listener at localhost/42919 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:55,621 INFO [Listener at localhost/42919 {}] util.FSUtils(462): Created version file at hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 with version=8 2023-11-19T01:29:55,622 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging 2023-11-19T01:29:55,638 DEBUG [Listener at localhost/42919 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-19T01:29:55,638 DEBUG [Listener at localhost/42919 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-19T01:29:55,638 DEBUG [Listener at localhost/42919 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-19T01:29:55,638 DEBUG [Listener at localhost/42919 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-19T01:29:55,857 DEBUG [Listener at localhost/42919 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-19T01:29:56,628 INFO [Listener at localhost/42919 {}] client.ConnectionUtils(128): master/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:29:56,675 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:29:56,675 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:29:56,675 INFO [Listener at localhost/42919 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:29:56,676 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:29:56,676 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:29:56,832 INFO [Listener at localhost/42919 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:29:56,972 INFO [Listener at localhost/42919 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-19T01:29:56,991 DEBUG [Listener at localhost/42919 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-19T01:29:56,997 INFO [Listener at localhost/42919 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:29:57,046 DEBUG [Listener at localhost/42919 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 14398 (auto-detected) 2023-11-19T01:29:57,048 DEBUG [Listener at localhost/42919 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:55:ff:fe:37:d3:4d (auto-detected) 2023-11-19T01:29:57,085 INFO [Listener at localhost/42919 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:34461 2023-11-19T01:29:57,100 INFO [Listener at localhost/42919 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:57,102 INFO [Listener at localhost/42919 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:57,125 INFO [Listener at localhost/42919 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:34461 connecting to ZooKeeper ensemble=127.0.0.1:53145 2023-11-19T01:29:57,176 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:344610x0, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:29:57,180 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:34461-0x10123a6de220000 connected 2023-11-19T01:29:57,213 DEBUG [Listener at localhost/42919 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:29:57,217 DEBUG [Listener at localhost/42919 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:29:57,220 DEBUG [Listener at localhost/42919 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:29:57,231 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34461 2023-11-19T01:29:57,231 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34461 2023-11-19T01:29:57,232 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34461 2023-11-19T01:29:57,232 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34461 2023-11-19T01:29:57,232 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34461 2023-11-19T01:29:57,286 INFO [Listener at localhost/42919 {}] log.Log(170): Logging initialized @7119ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-19T01:29:57,407 INFO [Listener at localhost/42919 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:29:57,408 INFO [Listener at localhost/42919 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:29:57,409 INFO [Listener at localhost/42919 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:29:57,411 INFO [Listener at localhost/42919 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-19T01:29:57,411 INFO [Listener at localhost/42919 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:29:57,411 INFO [Listener at localhost/42919 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:29:57,414 INFO [Listener at localhost/42919 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:29:57,547 INFO [Listener at localhost/42919 {}] http.HttpServer(1227): Jetty bound to port 40187 2023-11-19T01:29:57,549 INFO [Listener at localhost/42919 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:29:57,589 INFO [Listener at localhost/42919 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:29:57,592 INFO [Listener at localhost/42919 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@41083923{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:29:57,593 INFO [Listener at localhost/42919 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:29:57,593 INFO [Listener at localhost/42919 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3b083840{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:29:57,657 INFO [Listener at localhost/42919 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:29:57,671 INFO [Listener at localhost/42919 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:29:57,671 INFO [Listener at localhost/42919 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:29:57,673 INFO [Listener at localhost/42919 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-19T01:29:57,681 INFO [Listener at localhost/42919 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:29:58,820 INFO [Listener at localhost/42919 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@165763a1{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:29:58,830 INFO [Listener at localhost/42919 {}] server.AbstractConnector(333): Started ServerConnector@6417d641{HTTP/1.1, (http/1.1)}{0.0.0.0:40187} 2023-11-19T01:29:58,831 INFO [Listener at localhost/42919 {}] server.Server(415): Started @8663ms 2023-11-19T01:29:58,834 INFO [Listener at localhost/42919 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2, hbase.cluster.distributed=false 2023-11-19T01:29:58,895 INFO [Listener at localhost/42919 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:29:58,896 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:29:58,896 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:29:58,896 INFO [Listener at localhost/42919 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:29:58,896 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:29:58,896 INFO [Listener at localhost/42919 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:29:58,906 INFO [Listener at localhost/42919 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:29:58,908 INFO [Listener at localhost/42919 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:29:58,910 INFO [Listener at localhost/42919 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:41675 2023-11-19T01:29:58,912 INFO [Listener at localhost/42919 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-19T01:29:58,920 DEBUG [Listener at localhost/42919 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-19T01:29:58,920 INFO [Listener at localhost/42919 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:58,923 INFO [Listener at localhost/42919 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:58,926 INFO [Listener at localhost/42919 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:41675 connecting to ZooKeeper ensemble=127.0.0.1:53145 2023-11-19T01:29:58,930 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:416750x0, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:29:58,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:41675-0x10123a6de220001 connected 2023-11-19T01:29:58,931 DEBUG [Listener at localhost/42919 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:29:58,933 DEBUG [Listener at localhost/42919 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:29:58,934 DEBUG [Listener at localhost/42919 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:29:58,935 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41675 2023-11-19T01:29:58,935 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41675 2023-11-19T01:29:58,935 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41675 2023-11-19T01:29:58,936 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41675 2023-11-19T01:29:58,937 DEBUG [Listener at localhost/42919 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41675 2023-11-19T01:29:58,940 INFO [Listener at localhost/42919 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:29:58,941 INFO [Listener at localhost/42919 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:29:58,941 INFO [Listener at localhost/42919 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:29:58,941 INFO [Listener at localhost/42919 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-19T01:29:58,942 INFO [Listener at localhost/42919 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:29:58,942 INFO [Listener at localhost/42919 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:29:58,942 INFO [Listener at localhost/42919 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:29:58,944 INFO [Listener at localhost/42919 {}] http.HttpServer(1227): Jetty bound to port 38625 2023-11-19T01:29:58,944 INFO [Listener at localhost/42919 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:29:58,946 INFO [Listener at localhost/42919 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:29:58,946 INFO [Listener at localhost/42919 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@f5e5cc6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:29:58,947 INFO [Listener at localhost/42919 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:29:58,947 INFO [Listener at localhost/42919 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@328c6d9b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:29:58,959 INFO [Listener at localhost/42919 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:29:58,960 INFO [Listener at localhost/42919 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:29:58,960 INFO [Listener at localhost/42919 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:29:58,960 INFO [Listener at localhost/42919 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-19T01:29:58,961 INFO [Listener at localhost/42919 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:29:58,966 INFO [Listener at localhost/42919 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@35ed2944{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:29:58,967 INFO [Listener at localhost/42919 {}] server.AbstractConnector(333): Started ServerConnector@4fc7eff7{HTTP/1.1, (http/1.1)}{0.0.0.0:38625} 2023-11-19T01:29:58,967 INFO [Listener at localhost/42919 {}] server.Server(415): Started @8800ms 2023-11-19T01:29:58,972 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:29:58,975 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@5df9048b{HTTP/1.1, (http/1.1)}{0.0.0.0:41447} 2023-11-19T01:29:58,975 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(415): Started @8808ms 2023-11-19T01:29:58,975 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:29:58,988 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:29:58,988 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:29:58,991 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:29:58,998 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase6:34461 2023-11-19T01:29:59,014 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:29:59,014 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:29:59,014 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:29:59,014 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:29:59,015 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:29:59,017 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase6.apache.org,34461,1700357395839 from backup master directory 2023-11-19T01:29:59,017 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:29:59,021 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:29:59,021 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:29:59,021 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:29:59,021 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:29:59,022 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:29:59,025 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-19T01:29:59,027 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-19T01:29:59,539 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase.id with ID: 4dbb76a4-a25d-40ce-b8c8-8fbb4e008b17 2023-11-19T01:29:59,581 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:29:59,596 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:29:59,596 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:29:59,638 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:59,641 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-19T01:29:59,659 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-19T01:29:59,660 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-19T01:29:59,661 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:59,671 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:59,673 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:29:59,743 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store 2023-11-19T01:29:59,767 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-19T01:29:59,767 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:59,768 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:29:59,768 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:59,768 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:59,768 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:29:59,768 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:59,769 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:59,769 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:29:59,771 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/.initializing 2023-11-19T01:29:59,771 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/WALs/jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:29:59,780 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:29:59,794 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C34461%2C1700357395839, suffix=, logDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/WALs/jenkins-hbase6.apache.org,34461,1700357395839, archiveDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/oldWALs, maxLogs=10 2023-11-19T01:29:59,825 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/WALs/jenkins-hbase6.apache.org,34461,1700357395839/jenkins-hbase6.apache.org%2C34461%2C1700357395839.1700357399801, exclude list is [], retry=0 2023-11-19T01:29:59,854 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:29:59,861 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:59,931 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/WALs/jenkins-hbase6.apache.org,34461,1700357395839/jenkins-hbase6.apache.org%2C34461%2C1700357395839.1700357399801 2023-11-19T01:29:59,931 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:29:59,932 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:59,932 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:59,936 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:29:59,938 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-19T01:30:00,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:00,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-19T01:30:00,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:00,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-19T01:30:00,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:00,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-19T01:30:00,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:00,065 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,066 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,082 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-19T01:30:00,086 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:00,090 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:00,091 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11314666880, jitterRate=0.05376046895980835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-19T01:30:00,097 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:30:00,098 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-19T01:30:00,133 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d693da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:00,188 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-19T01:30:00,210 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-19T01:30:00,210 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-19T01:30:00,213 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-19T01:30:00,215 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-19T01:30:00,223 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 8 msec 2023-11-19T01:30:00,223 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-19T01:30:00,263 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-19T01:30:00,280 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-19T01:30:00,282 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-19T01:30:00,285 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-19T01:30:00,287 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-19T01:30:00,292 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-19T01:30:00,295 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-19T01:30:00,302 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-19T01:30:00,303 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-19T01:30:00,304 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-19T01:30:00,305 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-19T01:30:00,320 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-19T01:30:00,322 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-19T01:30:00,328 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:30:00,328 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:30:00,328 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,328 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,329 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase6.apache.org,34461,1700357395839, sessionid=0x10123a6de220000, setting cluster-up flag (Was=false) 2023-11-19T01:30:00,338 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-19T01:30:00,348 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,348 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,357 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-19T01:30:00,359 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:30:00,362 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,362 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,372 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-19T01:30:00,373 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:30:00,379 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,379 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:00,384 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-19T01:30:00,385 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:30:00,495 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase6:41675 2023-11-19T01:30:00,497 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(997): ClusterId : 4dbb76a4-a25d-40ce-b8c8-8fbb4e008b17 2023-11-19T01:30:00,499 INFO [RS:0;jenkins-hbase6:41675 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-19T01:30:00,501 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-19T01:30:00,508 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-19T01:30:00,508 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-19T01:30:00,512 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-19T01:30:00,512 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-19T01:30:00,515 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-19T01:30:00,516 DEBUG [RS:0;jenkins-hbase6:41675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@437de0aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:00,518 DEBUG [RS:0;jenkins-hbase6:41675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412e052d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:30:00,525 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-19T01:30:00,525 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-19T01:30:00,525 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-19T01:30:00,531 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,34461,1700357395839 with isa=jenkins-hbase6.apache.org/172.31.2.20:41675, startcode=1700357398894 2023-11-19T01:30:00,537 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-19T01:30:00,550 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-19T01:30:00,557 DEBUG [RS:0;jenkins-hbase6:41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:30:00,558 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-19T01:30:00,571 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase6.apache.org,34461,1700357395839 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-19T01:30:00,576 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:00,577 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:00,577 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:00,577 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:00,577 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase6:0, corePoolSize=10, maxPoolSize=10 2023-11-19T01:30:00,577 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,577 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:30:00,578 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,581 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1700357430581 2023-11-19T01:30:00,583 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-19T01:30:00,589 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-19T01:30:00,599 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:30:00,599 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-19T01:30:00,605 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-19T01:30:00,606 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-19T01:30:00,606 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,606 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-19T01:30:00,607 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-19T01:30:00,607 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-19T01:30:00,615 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,617 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-19T01:30:00,606 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:30:00,620 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-19T01:30:00,621 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-19T01:30:00,638 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c69ecbb to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:00,644 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45197, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:30:00,649 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-19T01:30:00,650 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:00,654 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a71c557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:00,655 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:30:00,656 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34461 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-19T01:30:00,659 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-19T01:30:00,660 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-19T01:30:00,668 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357400663,5,FailOnTimeoutGroup] 2023-11-19T01:30:00,671 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357400668,5,FailOnTimeoutGroup] 2023-11-19T01:30:00,671 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,672 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-19T01:30:00,674 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,675 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:00,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:30:00,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:30:00,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:00,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:30:00,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:30:00,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:00,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:30:00,691 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:30:00,691 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:00,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740 2023-11-19T01:30:00,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740 2023-11-19T01:30:00,697 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:30:00,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:30:00,704 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-19T01:30:00,704 WARN [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-19T01:30:00,704 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:00,705 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11797984320, jitterRate=0.09877291321754456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:30:00,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:30:00,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:30:00,708 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:30:00,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:30:00,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:30:00,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:30:00,710 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:30:00,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:30:00,714 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:30:00,714 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-19T01:30:00,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-19T01:30:00,745 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-19T01:30:00,749 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-19T01:30:00,805 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,34461,1700357395839 with isa=jenkins-hbase6.apache.org/172.31.2.20:41675, startcode=1700357398894 2023-11-19T01:30:00,811 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34461 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:00,822 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:00,823 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38223 2023-11-19T01:30:00,823 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=40187 2023-11-19T01:30:00,833 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-19T01:30:00,834 DEBUG [RS:0;jenkins-hbase6:41675 {}] zookeeper.ZKUtil(111): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:00,834 WARN [RS:0;jenkins-hbase6:41675 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:30:00,834 INFO [RS:0;jenkins-hbase6:41675 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:30:00,835 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:00,837 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase6.apache.org,41675,1700357398894] 2023-11-19T01:30:00,854 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-19T01:30:00,869 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-19T01:30:00,886 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-19T01:30:00,891 INFO [RS:0;jenkins-hbase6:41675 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-19T01:30:00,891 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,892 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-19T01:30:00,900 WARN [jenkins-hbase6:34461 {}] assignment.AssignmentManager(2298): No servers available; cannot place 1 unassigned regions. 2023-11-19T01:30:00,902 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,902 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,902 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,903 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,904 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,904 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:00,904 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:30:00,904 DEBUG [RS:0;jenkins-hbase6:41675 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:30:00,905 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,905 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,905 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,905 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,905 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,41675,1700357398894-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:30:00,921 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-19T01:30:00,924 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,41675,1700357398894-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:00,941 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.Replication(204): jenkins-hbase6.apache.org,41675,1700357398894 started 2023-11-19T01:30:00,941 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase6.apache.org,41675,1700357398894, RpcServer on jenkins-hbase6.apache.org/172.31.2.20:41675, sessionid=0x10123a6de220001 2023-11-19T01:30:00,941 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-19T01:30:00,941 DEBUG [RS:0;jenkins-hbase6:41675 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:00,941 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,41675,1700357398894' 2023-11-19T01:30:00,942 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-19T01:30:00,942 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-19T01:30:00,943 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-19T01:30:00,943 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-19T01:30:00,943 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,41675,1700357398894' 2023-11-19T01:30:00,943 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:00,944 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:00,944 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-19T01:30:00,945 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-19T01:30:00,945 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-19T01:30:00,945 DEBUG [RS:0;jenkins-hbase6:41675 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:00,945 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,41675,1700357398894' 2023-11-19T01:30:00,945 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-19T01:30:00,945 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-19T01:30:00,946 DEBUG [RS:0;jenkins-hbase6:41675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-19T01:30:00,946 INFO [RS:0;jenkins-hbase6:41675 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-19T01:30:00,946 INFO [RS:0;jenkins-hbase6:41675 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-19T01:30:01,057 INFO [RS:0;jenkins-hbase6:41675 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:30:01,060 INFO [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C41675%2C1700357398894, suffix=, logDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894, archiveDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/oldWALs, maxLogs=32 2023-11-19T01:30:01,073 DEBUG [RS:0;jenkins-hbase6:41675 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357401062, exclude list is [], retry=0 2023-11-19T01:30:01,079 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:30:01,083 INFO [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357401062 2023-11-19T01:30:01,083 DEBUG [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:30:01,152 DEBUG [jenkins-hbase6:34461 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-19T01:30:01,161 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:01,167 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,41675,1700357398894, state=OPENING 2023-11-19T01:30:01,176 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-19T01:30:01,177 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:01,177 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:01,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:30:01,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:30:01,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:01,389 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:01,391 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-19T01:30:01,395 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:59140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-19T01:30:01,407 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-19T01:30:01,408 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:30:01,409 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-19T01:30:01,412 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta, suffix=.meta, logDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894, archiveDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/oldWALs, maxLogs=32 2023-11-19T01:30:01,425 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357401413.meta, exclude list is [], retry=0 2023-11-19T01:30:01,430 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:30:01,437 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357401413.meta 2023-11-19T01:30:01,437 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:30:01,437 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:01,444 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:01,445 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-19T01:30:01,467 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-19T01:30:01,467 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-19T01:30:01,474 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-19T01:30:01,474 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:01,474 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-19T01:30:01,475 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-19T01:30:01,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:30:01,481 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:30:01,481 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:01,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:01,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:30:01,484 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:30:01,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:01,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:01,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:30:01,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:30:01,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:01,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:01,490 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740 2023-11-19T01:30:01,493 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740 2023-11-19T01:30:01,496 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:30:01,499 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:30:01,500 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10530811840, jitterRate=-0.01924172043800354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:30:01,502 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:30:01,516 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1700357401380 2023-11-19T01:30:01,541 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:01,542 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-19T01:30:01,543 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-19T01:30:01,544 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,41675,1700357398894, state=OPEN 2023-11-19T01:30:01,548 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:30:01,548 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:30:01,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:30:01,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:30:01,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-19T01:30:01,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,41675,1700357398894 in 367 msec 2023-11-19T01:30:01,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-19T01:30:01,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 827 msec 2023-11-19T01:30:01,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.1210 sec 2023-11-19T01:30:01,572 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1700357401572, completionTime=-1 2023-11-19T01:30:01,572 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-19T01:30:01,572 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-19T01:30:01,642 DEBUG [hconnection-0x1cd2e91b-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:01,645 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:59154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:01,661 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-19T01:30:01,661 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1700357461661 2023-11-19T01:30:01,661 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1700357521661 2023-11-19T01:30:01,661 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 88 msec 2023-11-19T01:30:01,686 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34461,1700357395839-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:01,686 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34461,1700357395839-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:01,686 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34461,1700357395839-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:01,689 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase6:34461, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:01,689 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:01,696 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-19T01:30:01,710 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-19T01:30:01,711 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:30:01,721 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-19T01:30:01,725 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:01,727 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:01,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:02,150 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 92f2c176e8984c832db821343de7a411, NAME => 'hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:02,569 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:02,569 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 92f2c176e8984c832db821343de7a411, disabling compactions & flushes 2023-11-19T01:30:02,569 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,569 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,569 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. after waiting 0 ms 2023-11-19T01:30:02,569 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,569 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,569 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 92f2c176e8984c832db821343de7a411: 2023-11-19T01:30:02,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:02,589 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1700357402574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357402574"}]},"ts":"1700357402574"} 2023-11-19T01:30:02,633 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:02,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:02,642 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357402636"}]},"ts":"1700357402636"} 2023-11-19T01:30:02,646 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-19T01:30:02,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=92f2c176e8984c832db821343de7a411, ASSIGN}] 2023-11-19T01:30:02,655 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=92f2c176e8984c832db821343de7a411, ASSIGN 2023-11-19T01:30:02,656 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=92f2c176e8984c832db821343de7a411, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:02,808 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=92f2c176e8984c832db821343de7a411, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:02,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 92f2c176e8984c832db821343de7a411, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:02,966 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:02,972 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,973 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 92f2c176e8984c832db821343de7a411, NAME => 'hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:02,974 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:02,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:02,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,977 INFO [StoreOpener-92f2c176e8984c832db821343de7a411-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,979 INFO [StoreOpener-92f2c176e8984c832db821343de7a411-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f2c176e8984c832db821343de7a411 columnFamilyName info 2023-11-19T01:30:02,980 DEBUG [StoreOpener-92f2c176e8984c832db821343de7a411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:02,980 INFO [StoreOpener-92f2c176e8984c832db821343de7a411-1 {}] regionserver.HStore(324): Store=92f2c176e8984c832db821343de7a411/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:02,982 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,983 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 92f2c176e8984c832db821343de7a411 2023-11-19T01:30:02,989 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:02,990 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 92f2c176e8984c832db821343de7a411; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11560218400, jitterRate=0.07662923634052277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:02,991 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 92f2c176e8984c832db821343de7a411: 2023-11-19T01:30:02,993 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411., pid=6, masterSystemTime=1700357402966 2023-11-19T01:30:02,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,997 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:30:02,998 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=92f2c176e8984c832db821343de7a411, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:03,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-19T01:30:03,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 92f2c176e8984c832db821343de7a411, server=jenkins-hbase6.apache.org,41675,1700357398894 in 189 msec 2023-11-19T01:30:03,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-19T01:30:03,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=92f2c176e8984c832db821343de7a411, ASSIGN in 353 msec 2023-11-19T01:30:03,014 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:03,014 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357403014"}]},"ts":"1700357403014"} 2023-11-19T01:30:03,017 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-19T01:30:03,021 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:03,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3090 sec 2023-11-19T01:30:03,026 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-19T01:30:03,032 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:03,032 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:03,032 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:30:03,095 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-19T01:30:03,119 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:03,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 43 msec 2023-11-19T01:30:03,132 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-19T01:30:03,146 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:03,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 19 msec 2023-11-19T01:30:03,169 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-19T01:30:03,173 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-19T01:30:03,176 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 4.153sec 2023-11-19T01:30:03,178 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-19T01:30:03,179 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-19T01:30:03,181 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-19T01:30:03,181 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-19T01:30:03,182 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-19T01:30:03,183 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34461,1700357395839-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:30:03,183 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34461,1700357395839-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-19T01:30:03,197 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-19T01:30:03,199 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-19T01:30:03,270 DEBUG [Listener at localhost/42919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x59761e0b to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:03,275 DEBUG [Listener at localhost/42919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4456e08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:03,281 DEBUG [Listener at localhost/42919 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-19T01:30:03,281 DEBUG [Listener at localhost/42919 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-19T01:30:03,296 DEBUG [hconnection-0x62549eb1-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:03,309 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:59162, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:03,322 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:30:03,334 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir so I do NOT create it in target/test-data/764eb130-6325-6a06-5313-0476a78fcd87 2023-11-19T01:30:03,334 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.tmp.dir so I do NOT create it in target/test-data/764eb130-6325-6a06-5313-0476a78fcd87 2023-11-19T01:30:03,334 DEBUG [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4, deleteOnExit=true 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/test.cache.data in system properties and HBase conf 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:30:03,335 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:30:03,335 DEBUG [Listener at localhost/42919 {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:30:03,336 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:30:03,337 INFO [Listener at localhost/42919 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-19T01:30:03,352 WARN [Listener at localhost/42919 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:30:03,353 WARN [Listener at localhost/42919 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:30:03,412 WARN [Listener at localhost/42919 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:03,416 INFO [Listener at localhost/42919 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:03,424 INFO [Listener at localhost/42919 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/java.io.tmpdir/Jetty_localhost_43523_hdfs____ja85q9/webapp 2023-11-19T01:30:03,525 INFO [Listener at localhost/42919 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43523 2023-11-19T01:30:03,541 WARN [Listener at localhost/42919 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:30:03,541 WARN [Listener at localhost/42919 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:30:03,600 WARN [Listener at localhost/34159 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:03,627 WARN [Listener at localhost/34159 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:30:03,632 WARN [Listener at localhost/34159 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:03,634 INFO [Listener at localhost/34159 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:03,646 INFO [Listener at localhost/34159 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/java.io.tmpdir/Jetty_localhost_34921_datanode____hqkuqm/webapp 2023-11-19T01:30:03,769 INFO [Listener at localhost/34159 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34921 2023-11-19T01:30:03,798 WARN [Listener at localhost/46541 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:03,823 WARN [Listener at localhost/46541 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:30:03,826 WARN [Listener at localhost/46541 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:03,828 INFO [Listener at localhost/46541 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:03,831 INFO [Listener at localhost/46541 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/java.io.tmpdir/Jetty_localhost_36529_datanode____4d6hy8/webapp 2023-11-19T01:30:03,884 WARN [Thread-202 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:30:03,922 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x950d7a2ee20fa5ac: Processing first storage report for DS-9b8e1f4a-b201-4b5b-9e99-aa0bb5691267 from datanode b5960f6b-4eec-41f3-a486-e55d15795eda 2023-11-19T01:30:03,923 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x950d7a2ee20fa5ac: from storage DS-9b8e1f4a-b201-4b5b-9e99-aa0bb5691267 node DatanodeRegistration(127.0.0.1:36721, datanodeUuid=b5960f6b-4eec-41f3-a486-e55d15795eda, infoPort=33387, infoSecurePort=0, ipcPort=46541, storageInfo=lv=-57;cid=testClusterID;nsid=1606804758;c=1700357403356), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:03,923 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x950d7a2ee20fa5ac: Processing first storage report for DS-29b49e66-74dc-467e-ba9b-bad011bf0bcd from datanode b5960f6b-4eec-41f3-a486-e55d15795eda 2023-11-19T01:30:03,923 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x950d7a2ee20fa5ac: from storage DS-29b49e66-74dc-467e-ba9b-bad011bf0bcd node DatanodeRegistration(127.0.0.1:36721, datanodeUuid=b5960f6b-4eec-41f3-a486-e55d15795eda, infoPort=33387, infoSecurePort=0, ipcPort=46541, storageInfo=lv=-57;cid=testClusterID;nsid=1606804758;c=1700357403356), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-19T01:30:03,949 INFO [Listener at localhost/46541 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36529 2023-11-19T01:30:03,960 WARN [Listener at localhost/41583 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:03,991 WARN [Listener at localhost/41583 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:30:03,994 WARN [Listener at localhost/41583 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:03,995 INFO [Listener at localhost/41583 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:04,007 INFO [Listener at localhost/41583 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/java.io.tmpdir/Jetty_localhost_33837_datanode____yszm77/webapp 2023-11-19T01:30:04,057 WARN [Thread-237 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:30:04,089 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x86308e080d9d9469: Processing first storage report for DS-5a48e3fc-1fb1-497d-b751-141adc86f47b from datanode 49555abc-4d33-4685-be34-f5c5aeea2cee 2023-11-19T01:30:04,090 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x86308e080d9d9469: from storage DS-5a48e3fc-1fb1-497d-b751-141adc86f47b node DatanodeRegistration(127.0.0.1:35453, datanodeUuid=49555abc-4d33-4685-be34-f5c5aeea2cee, infoPort=37321, infoSecurePort=0, ipcPort=41583, storageInfo=lv=-57;cid=testClusterID;nsid=1606804758;c=1700357403356), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:04,090 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x86308e080d9d9469: Processing first storage report for DS-1cf7f5e1-8e31-4093-bef4-bc4a56fd90b5 from datanode 49555abc-4d33-4685-be34-f5c5aeea2cee 2023-11-19T01:30:04,090 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x86308e080d9d9469: from storage DS-1cf7f5e1-8e31-4093-bef4-bc4a56fd90b5 node DatanodeRegistration(127.0.0.1:35453, datanodeUuid=49555abc-4d33-4685-be34-f5c5aeea2cee, infoPort=37321, infoSecurePort=0, ipcPort=41583, storageInfo=lv=-57;cid=testClusterID;nsid=1606804758;c=1700357403356), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:04,122 INFO [Listener at localhost/41583 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33837 2023-11-19T01:30:04,134 WARN [Listener at localhost/34903 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:04,219 WARN [Thread-271 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:30:04,250 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xceb80aa3f8b861ed: Processing first storage report for DS-c9a17779-b0ec-4156-8f44-7dc4e65bf6fa from datanode dd141294-c327-4d14-9c0c-b205fa3ed59a 2023-11-19T01:30:04,251 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xceb80aa3f8b861ed: from storage DS-c9a17779-b0ec-4156-8f44-7dc4e65bf6fa node DatanodeRegistration(127.0.0.1:37711, datanodeUuid=dd141294-c327-4d14-9c0c-b205fa3ed59a, infoPort=41693, infoSecurePort=0, ipcPort=34903, storageInfo=lv=-57;cid=testClusterID;nsid=1606804758;c=1700357403356), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-19T01:30:04,251 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xceb80aa3f8b861ed: Processing first storage report for DS-3a1ab065-0fb7-4d0d-81cf-ff2aee76b869 from datanode dd141294-c327-4d14-9c0c-b205fa3ed59a 2023-11-19T01:30:04,251 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xceb80aa3f8b861ed: from storage DS-3a1ab065-0fb7-4d0d-81cf-ff2aee76b869 node DatanodeRegistration(127.0.0.1:37711, datanodeUuid=dd141294-c327-4d14-9c0c-b205fa3ed59a, infoPort=41693, infoSecurePort=0, ipcPort=34903, storageInfo=lv=-57;cid=testClusterID;nsid=1606804758;c=1700357403356), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:04,256 DEBUG [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87 2023-11-19T01:30:04,256 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-19T01:30:04,256 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(1143): NOT STARTING DFS 2023-11-19T01:30:04,256 INFO [Listener at localhost/34903 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:04,257 INFO [Listener at localhost/34903 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:04,680 INFO [Listener at localhost/34903 {}] util.FSUtils(462): Created version file at hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7 with version=8 2023-11-19T01:30:04,680 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging 2023-11-19T01:30:04,682 DEBUG [Listener at localhost/34903 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-19T01:30:04,682 DEBUG [Listener at localhost/34903 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-19T01:30:04,682 DEBUG [Listener at localhost/34903 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-19T01:30:04,682 DEBUG [Listener at localhost/34903 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-19T01:30:04,684 INFO [Listener at localhost/34903 {}] client.ConnectionUtils(128): master/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:30:04,684 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:30:04,684 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:30:04,684 INFO [Listener at localhost/34903 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:30:04,685 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:30:04,685 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:30:04,685 INFO [Listener at localhost/34903 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:30:04,685 INFO [Listener at localhost/34903 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:30:04,687 INFO [Listener at localhost/34903 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:42885 2023-11-19T01:30:04,687 INFO [Listener at localhost/34903 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:04,689 INFO [Listener at localhost/34903 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:04,691 INFO [Listener at localhost/34903 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:42885 connecting to ZooKeeper ensemble=127.0.0.1:53145 2023-11-19T01:30:04,695 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:428850x0, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:30:04,696 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:42885-0x10123a6de220004 connected 2023-11-19T01:30:04,710 DEBUG [Listener at localhost/34903 {}] zookeeper.ZKUtil(113): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-11-19T01:30:04,710 DEBUG [Listener at localhost/34903 {}] zookeeper.ZKUtil(113): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-19T01:30:04,711 DEBUG [Listener at localhost/34903 {}] zookeeper.ZKUtil(113): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-11-19T01:30:04,712 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42885 2023-11-19T01:30:04,713 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42885 2023-11-19T01:30:04,713 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42885 2023-11-19T01:30:04,713 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42885 2023-11-19T01:30:04,713 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42885 2023-11-19T01:30:04,717 INFO [Listener at localhost/34903 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:30:04,717 INFO [Listener at localhost/34903 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:30:04,717 INFO [Listener at localhost/34903 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:30:04,718 INFO [Listener at localhost/34903 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-19T01:30:04,718 INFO [Listener at localhost/34903 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:30:04,718 INFO [Listener at localhost/34903 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:30:04,718 INFO [Listener at localhost/34903 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:30:04,720 INFO [Listener at localhost/34903 {}] http.HttpServer(1227): Jetty bound to port 41291 2023-11-19T01:30:04,720 INFO [Listener at localhost/34903 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:30:04,727 INFO [Listener at localhost/34903 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:30:04,727 INFO [Listener at localhost/34903 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2c81789e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:30:04,728 INFO [Listener at localhost/34903 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:30:04,728 INFO [Listener at localhost/34903 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6eb2633e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:30:04,741 INFO [Listener at localhost/34903 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:30:04,742 INFO [Listener at localhost/34903 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:30:04,743 INFO [Listener at localhost/34903 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:30:04,743 INFO [Listener at localhost/34903 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-19T01:30:04,744 INFO [Listener at localhost/34903 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:30:04,854 INFO [Listener at localhost/34903 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@1fd325fa{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:30:04,855 INFO [Listener at localhost/34903 {}] server.AbstractConnector(333): Started ServerConnector@4597b7cc{HTTP/1.1, (http/1.1)}{0.0.0.0:41291} 2023-11-19T01:30:04,855 INFO [Listener at localhost/34903 {}] server.Server(415): Started @14688ms 2023-11-19T01:30:04,856 INFO [Listener at localhost/34903 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7, hbase.cluster.distributed=false 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:30:04,871 INFO [Listener at localhost/34903 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:30:04,872 INFO [Listener at localhost/34903 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:30:04,873 INFO [Listener at localhost/34903 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:33353 2023-11-19T01:30:04,874 INFO [Listener at localhost/34903 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-19T01:30:04,875 DEBUG [Listener at localhost/34903 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-19T01:30:04,876 INFO [Listener at localhost/34903 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:04,877 INFO [Listener at localhost/34903 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:04,879 INFO [Listener at localhost/34903 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:33353 connecting to ZooKeeper ensemble=127.0.0.1:53145 2023-11-19T01:30:04,883 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:333530x0, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:30:04,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:33353-0x10123a6de220005 connected 2023-11-19T01:30:04,885 DEBUG [Listener at localhost/34903 {}] zookeeper.ZKUtil(113): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-11-19T01:30:04,886 DEBUG [Listener at localhost/34903 {}] zookeeper.ZKUtil(113): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-19T01:30:04,886 DEBUG [Listener at localhost/34903 {}] zookeeper.ZKUtil(113): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/acl 2023-11-19T01:30:04,890 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33353 2023-11-19T01:30:04,891 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33353 2023-11-19T01:30:04,891 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33353 2023-11-19T01:30:04,893 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33353 2023-11-19T01:30:04,893 DEBUG [Listener at localhost/34903 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33353 2023-11-19T01:30:04,896 INFO [Listener at localhost/34903 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:30:04,896 INFO [Listener at localhost/34903 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:30:04,897 INFO [Listener at localhost/34903 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:30:04,897 INFO [Listener at localhost/34903 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-19T01:30:04,897 INFO [Listener at localhost/34903 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:30:04,897 INFO [Listener at localhost/34903 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:30:04,897 INFO [Listener at localhost/34903 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:30:04,898 INFO [Listener at localhost/34903 {}] http.HttpServer(1227): Jetty bound to port 45721 2023-11-19T01:30:04,898 INFO [Listener at localhost/34903 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:30:04,911 INFO [Listener at localhost/34903 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:30:04,911 INFO [Listener at localhost/34903 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@700ecd02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:30:04,912 INFO [Listener at localhost/34903 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:30:04,912 INFO [Listener at localhost/34903 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6870b91f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:30:04,919 INFO [Listener at localhost/34903 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:30:04,920 INFO [Listener at localhost/34903 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:30:04,920 INFO [Listener at localhost/34903 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:30:04,921 INFO [Listener at localhost/34903 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-19T01:30:04,923 INFO [Listener at localhost/34903 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:30:04,924 INFO [Listener at localhost/34903 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@80d046f{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:30:04,925 INFO [Listener at localhost/34903 {}] server.AbstractConnector(333): Started ServerConnector@1dc53c84{HTTP/1.1, (http/1.1)}{0.0.0.0:45721} 2023-11-19T01:30:04,925 INFO [Listener at localhost/34903 {}] server.Server(415): Started @14757ms 2023-11-19T01:30:04,926 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:30:04,930 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@353b8c75{HTTP/1.1, (http/1.1)}{0.0.0.0:34653} 2023-11-19T01:30:04,930 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(415): Started @14763ms 2023-11-19T01:30:04,930 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /2/backup-masters/jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:04,934 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-19T01:30:04,934 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-19T01:30:04,934 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on existing znode=/2/backup-masters/jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:04,937 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-11-19T01:30:04,937 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:04,938 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/master 2023-11-19T01:30:04,938 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:04,939 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on existing znode=/2/master 2023-11-19T01:30:04,940 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /2/backup-masters/jenkins-hbase6.apache.org,42885,1700357404683 from backup master directory 2023-11-19T01:30:04,943 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase6:42885 2023-11-19T01:30:04,945 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/backup-masters/jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:04,945 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-19T01:30:04,945 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/backup-masters 2023-11-19T01:30:04,945 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:30:04,945 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:04,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on existing znode=/2/master 2023-11-19T01:30:05,374 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/hbase.id with ID: 399cd594-e61c-4b0c-99a6-77a4c08c7a12 2023-11-19T01:30:05,395 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:05,399 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,399 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,416 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:05,416 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-19T01:30:05,417 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:30:05,434 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store 2023-11-19T01:30:05,848 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:05,848 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:30:05,848 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:05,848 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:05,848 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:30:05,848 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:05,848 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:05,849 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:30:05,850 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/.initializing 2023-11-19T01:30:05,850 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:34159/tmp/wal/MasterData/WALs/jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:05,852 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:30:05,855 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C42885%2C1700357404683, suffix=, logDir=hdfs://localhost:34159/tmp/wal/MasterData/WALs/jenkins-hbase6.apache.org,42885,1700357404683, archiveDir=hdfs://localhost:34159/tmp/wal/MasterData/oldWALs, maxLogs=10 2023-11-19T01:30:05,866 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/MasterData/WALs/jenkins-hbase6.apache.org,42885,1700357404683/jenkins-hbase6.apache.org%2C42885%2C1700357404683.1700357405855, exclude list is [], retry=0 2023-11-19T01:30:05,872 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36721,DS-9b8e1f4a-b201-4b5b-9e99-aa0bb5691267,DISK] 2023-11-19T01:30:05,880 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/MasterData/WALs/jenkins-hbase6.apache.org,42885,1700357404683/jenkins-hbase6.apache.org%2C42885%2C1700357404683.1700357405855 2023-11-19T01:30:05,880 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36721,DS-9b8e1f4a-b201-4b5b-9e99-aa0bb5691267,DISK]] 2023-11-19T01:30:05,880 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:05,880 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:05,881 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,881 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-19T01:30:05,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:05,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:05,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-19T01:30:05,892 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:05,893 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:05,893 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-19T01:30:05,895 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:05,896 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:05,896 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-19T01:30:05,899 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:05,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:05,901 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,901 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,902 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,905 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-19T01:30:05,907 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:30:05,911 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34159/tmp/wal/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:05,912 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10000613280, jitterRate=-0.06862030923366547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-19T01:30:05,912 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:30:05,913 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-19T01:30:05,918 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adab1b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:05,919 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-19T01:30:05,920 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-19T01:30:05,920 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-19T01:30:05,920 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-19T01:30:05,921 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 0 msec 2023-11-19T01:30:05,921 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 0 msec 2023-11-19T01:30:05,921 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-19T01:30:05,924 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-19T01:30:05,925 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Unable to get data of znode /2/balancer because node does not exist (not necessarily an error) 2023-11-19T01:30:05,927 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/balancer already deleted, retry=false 2023-11-19T01:30:05,927 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-19T01:30:05,928 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Unable to get data of znode /2/normalizer because node does not exist (not necessarily an error) 2023-11-19T01:30:05,929 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/normalizer already deleted, retry=false 2023-11-19T01:30:05,929 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-19T01:30:05,930 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Unable to get data of znode /2/switch/split because node does not exist (not necessarily an error) 2023-11-19T01:30:05,932 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/split already deleted, retry=false 2023-11-19T01:30:05,933 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Unable to get data of znode /2/switch/merge because node does not exist (not necessarily an error) 2023-11-19T01:30:05,934 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/switch/merge already deleted, retry=false 2023-11-19T01:30:05,935 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Unable to get data of znode /2/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-19T01:30:05,936 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /2/snapshot-cleanup already deleted, retry=false 2023-11-19T01:30:05,943 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-11-19T01:30:05,943 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/running 2023-11-19T01:30:05,943 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,943 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,944 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase6.apache.org,42885,1700357404683, sessionid=0x10123a6de220004, setting cluster-up flag (Was=false) 2023-11-19T01:30:05,944 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-19T01:30:05,947 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,947 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,954 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/flush-table-proc/acquired, /2/flush-table-proc/reached, /2/flush-table-proc/abort 2023-11-19T01:30:05,955 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:05,958 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,958 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,966 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/rolllog-proc/acquired, /2/rolllog-proc/reached, /2/rolllog-proc/abort 2023-11-19T01:30:05,967 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:05,972 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,972 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:05,978 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /2/online-snapshot/acquired, /2/online-snapshot/reached, /2/online-snapshot/abort 2023-11-19T01:30:05,980 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:05,984 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-19T01:30:05,984 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-19T01:30:05,984 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-19T01:30:05,985 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase6.apache.org,42885,1700357404683 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-19T01:30:05,986 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:05,986 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:05,986 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:05,986 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:30:05,986 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase6:0, corePoolSize=10, maxPoolSize=10 2023-11-19T01:30:05,986 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:05,987 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:30:05,987 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1700357435993 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-19T01:30:05,994 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:05,995 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-19T01:30:05,996 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-19T01:30:05,996 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-19T01:30:05,996 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:30:05,996 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-19T01:30:05,998 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7fff1338 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:06,001 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:06,001 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:30:06,015 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78164969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:06,016 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:30:06,022 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-19T01:30:06,024 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-19T01:30:06,024 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357406024,5,FailOnTimeoutGroup] 2023-11-19T01:30:06,027 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357406024,5,FailOnTimeoutGroup] 2023-11-19T01:30:06,027 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,029 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-19T01:30:06,029 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,030 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,041 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase6:33353 2023-11-19T01:30:06,043 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(997): ClusterId : 399cd594-e61c-4b0c-99a6-77a4c08c7a12 2023-11-19T01:30:06,043 INFO [RS:0;jenkins-hbase6:33353 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-19T01:30:06,043 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-19T01:30:06,047 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-19T01:30:06,047 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-19T01:30:06,050 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-19T01:30:06,050 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-19T01:30:06,053 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-19T01:30:06,055 DEBUG [RS:0;jenkins-hbase6:33353 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7faf8f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:06,058 DEBUG [RS:0;jenkins-hbase6:33353 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772f1d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:30:06,058 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-19T01:30:06,058 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-19T01:30:06,058 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-19T01:30:06,059 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,42885,1700357404683 with isa=jenkins-hbase6.apache.org/172.31.2.20:33353, startcode=1700357404870 2023-11-19T01:30:06,060 DEBUG [RS:0;jenkins-hbase6:33353 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:30:06,064 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44219, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:30:06,065 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42885 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:06,068 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7 2023-11-19T01:30:06,069 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:34159 2023-11-19T01:30:06,069 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=41291 2023-11-19T01:30:06,072 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-11-19T01:30:06,072 DEBUG [RS:0;jenkins-hbase6:33353 {}] zookeeper.ZKUtil(111): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on existing znode=/2/rs/jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:06,073 WARN [RS:0;jenkins-hbase6:33353 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:30:06,073 INFO [RS:0;jenkins-hbase6:33353 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:30:06,073 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:34159/tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:06,073 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase6.apache.org,33353,1700357404870] 2023-11-19T01:30:06,080 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-19T01:30:06,080 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-19T01:30:06,084 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-19T01:30:06,084 INFO [RS:0;jenkins-hbase6:33353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-19T01:30:06,084 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,086 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-19T01:30:06,088 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:30:06,089 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,090 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,090 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,090 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,090 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:30:06,090 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:30:06,090 DEBUG [RS:0;jenkins-hbase6:33353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:30:06,095 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,095 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,095 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,096 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,096 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33353,1700357404870-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:30:06,107 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-19T01:30:06,107 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33353,1700357404870-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:06,118 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.Replication(204): jenkins-hbase6.apache.org,33353,1700357404870 started 2023-11-19T01:30:06,118 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase6.apache.org,33353,1700357404870, RpcServer on jenkins-hbase6.apache.org/172.31.2.20:33353, sessionid=0x10123a6de220005 2023-11-19T01:30:06,119 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-19T01:30:06,119 DEBUG [RS:0;jenkins-hbase6:33353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:06,119 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,33353,1700357404870' 2023-11-19T01:30:06,119 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/flush-table-proc/abort' 2023-11-19T01:30:06,119 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/flush-table-proc/acquired' 2023-11-19T01:30:06,120 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-19T01:30:06,120 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-19T01:30:06,120 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,33353,1700357404870' 2023-11-19T01:30:06,120 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/rolllog-proc/abort' 2023-11-19T01:30:06,121 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/rolllog-proc/acquired' 2023-11-19T01:30:06,121 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-19T01:30:06,121 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-19T01:30:06,121 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-19T01:30:06,121 DEBUG [RS:0;jenkins-hbase6:33353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:06,121 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,33353,1700357404870' 2023-11-19T01:30:06,121 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/2/online-snapshot/abort' 2023-11-19T01:30:06,122 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/2/online-snapshot/acquired' 2023-11-19T01:30:06,122 DEBUG [RS:0;jenkins-hbase6:33353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-19T01:30:06,122 INFO [RS:0;jenkins-hbase6:33353 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-19T01:30:06,122 INFO [RS:0;jenkins-hbase6:33353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-19T01:30:06,223 INFO [RS:0;jenkins-hbase6:33353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:30:06,226 INFO [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C33353%2C1700357404870, suffix=, logDir=hdfs://localhost:34159/tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870, archiveDir=hdfs://localhost:34159/tmp/wal/oldWALs, maxLogs=32 2023-11-19T01:30:06,237 DEBUG [RS:0;jenkins-hbase6:33353 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870/jenkins-hbase6.apache.org%2C33353%2C1700357404870.1700357406227, exclude list is [], retry=0 2023-11-19T01:30:06,247 DEBUG [RS-EventLoopGroup-6-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36721,DS-9b8e1f4a-b201-4b5b-9e99-aa0bb5691267,DISK] 2023-11-19T01:30:06,255 INFO [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870/jenkins-hbase6.apache.org%2C33353%2C1700357404870.1700357406227 2023-11-19T01:30:06,256 DEBUG [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36721,DS-9b8e1f4a-b201-4b5b-9e99-aa0bb5691267,DISK]] 2023-11-19T01:30:06,436 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-19T01:30:06,436 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7 2023-11-19T01:30:06,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:06,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:30:06,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:30:06,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:06,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:06,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:30:06,871 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:30:06,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:06,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:06,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:30:06,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:30:06,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:06,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:06,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740 2023-11-19T01:30:06,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740 2023-11-19T01:30:06,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/tmp/wal/data/hbase/meta/1588230740 2023-11-19T01:30:06,882 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:30:06,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:30:06,887 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34159/tmp/wal/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:06,887 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10196816320, jitterRate=-0.050347477197647095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:30:06,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:30:06,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:30:06,888 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:30:06,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:30:06,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:30:06,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:30:06,888 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:30:06,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:30:06,890 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:30:06,890 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-19T01:30:06,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-19T01:30:06,892 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-19T01:30:06,895 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-19T01:30:07,010 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2023-11-19T01:30:07,046 DEBUG [jenkins-hbase6:42885 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-19T01:30:07,047 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:07,049 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,33353,1700357404870, state=OPENING 2023-11-19T01:30:07,051 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-19T01:30:07,053 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:07,053 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:07,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-19T01:30:07,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,33353,1700357404870}] 2023-11-19T01:30:07,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-19T01:30:07,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-19T01:30:07,114 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-19T01:30:07,115 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-19T01:30:07,115 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:07,117 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-19T01:30:07,117 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:07,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:07,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-19T01:30:07,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:30:07,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-19T01:30:07,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-19T01:30:07,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:07,210 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:07,210 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-19T01:30:07,213 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52986, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-19T01:30:07,218 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-19T01:30:07,218 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:30:07,219 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-19T01:30:07,221 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C33353%2C1700357404870.meta, suffix=.meta, logDir=hdfs://localhost:34159/tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870, archiveDir=hdfs://localhost:34159/tmp/wal/oldWALs, maxLogs=32 2023-11-19T01:30:07,234 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870/jenkins-hbase6.apache.org%2C33353%2C1700357404870.meta.1700357407222.meta, exclude list is [], retry=0 2023-11-19T01:30:07,240 DEBUG [RS-EventLoopGroup-6-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35453,DS-5a48e3fc-1fb1-497d-b751-141adc86f47b,DISK] 2023-11-19T01:30:07,243 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /tmp/wal/WALs/jenkins-hbase6.apache.org,33353,1700357404870/jenkins-hbase6.apache.org%2C33353%2C1700357404870.meta.1700357407222.meta 2023-11-19T01:30:07,243 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35453,DS-5a48e3fc-1fb1-497d-b751-141adc86f47b,DISK]] 2023-11-19T01:30:07,243 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:07,244 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:07,244 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-19T01:30:07,244 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-19T01:30:07,244 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-19T01:30:07,244 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-19T01:30:07,244 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:07,244 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-19T01:30:07,244 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-19T01:30:07,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:30:07,247 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:30:07,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:07,248 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:07,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:30:07,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:30:07,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:07,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:07,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:30:07,253 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:30:07,253 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:07,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:30:07,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740 2023-11-19T01:30:07,255 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740 2023-11-19T01:30:07,257 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/tmp/wal/data/hbase/meta/1588230740 2023-11-19T01:30:07,259 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:30:07,261 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:30:07,262 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9465101120, jitterRate=-0.11849376559257507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:30:07,263 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:30:07,264 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1700357407210 2023-11-19T01:30:07,268 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-19T01:30:07,268 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-19T01:30:07,269 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:07,270 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,33353,1700357404870, state=OPEN 2023-11-19T01:30:07,272 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-11-19T01:30:07,272 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/meta-region-server 2023-11-19T01:30:07,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-19T01:30:07,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /2/meta-region-server: CHANGED 2023-11-19T01:30:07,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-19T01:30:07,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,33353,1700357404870 in 218 msec 2023-11-19T01:30:07,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-19T01:30:07,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 385 msec 2023-11-19T01:30:07,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2980 sec 2023-11-19T01:30:07,281 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1700357407281, completionTime=-1 2023-11-19T01:30:07,281 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-19T01:30:07,281 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-19T01:30:07,283 DEBUG [hconnection-0x6b531ce6-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:07,286 INFO [RS-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52992, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:07,287 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-19T01:30:07,287 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1700357467287 2023-11-19T01:30:07,287 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1700357527287 2023-11-19T01:30:07,287 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 5 msec 2023-11-19T01:30:07,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,42885,1700357404683-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:07,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,42885,1700357404683-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:07,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,42885,1700357404683-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:07,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase6:42885, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:07,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-19T01:30:07,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-19T01:30:07,298 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:30:07,299 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-19T01:30:07,299 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-19T01:30:07,301 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:07,301 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:07,302 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:07,715 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8af47d28058a302c1449b0cfff8bfc3d, NAME => 'hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7 2023-11-19T01:30:07,727 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:07,728 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 8af47d28058a302c1449b0cfff8bfc3d, disabling compactions & flushes 2023-11-19T01:30:07,728 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:07,728 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:07,728 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. after waiting 0 ms 2023-11-19T01:30:07,728 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:07,728 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:07,728 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8af47d28058a302c1449b0cfff8bfc3d: 2023-11-19T01:30:07,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:07,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1700357407730"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357407730"}]},"ts":"1700357407730"} 2023-11-19T01:30:07,735 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:07,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:07,736 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357407736"}]},"ts":"1700357407736"} 2023-11-19T01:30:07,739 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-19T01:30:07,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8af47d28058a302c1449b0cfff8bfc3d, ASSIGN}] 2023-11-19T01:30:07,746 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8af47d28058a302c1449b0cfff8bfc3d, ASSIGN 2023-11-19T01:30:07,747 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8af47d28058a302c1449b0cfff8bfc3d, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33353,1700357404870; forceNewPlan=false, retain=false 2023-11-19T01:30:07,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8af47d28058a302c1449b0cfff8bfc3d, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:07,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8af47d28058a302c1449b0cfff8bfc3d, server=jenkins-hbase6.apache.org,33353,1700357404870}] 2023-11-19T01:30:08,054 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:08,060 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:08,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 8af47d28058a302c1449b0cfff8bfc3d, NAME => 'hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:08,060 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:08,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:08,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,063 INFO [StoreOpener-8af47d28058a302c1449b0cfff8bfc3d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,065 INFO [StoreOpener-8af47d28058a302c1449b0cfff8bfc3d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8af47d28058a302c1449b0cfff8bfc3d columnFamilyName info 2023-11-19T01:30:08,065 DEBUG [StoreOpener-8af47d28058a302c1449b0cfff8bfc3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:08,066 INFO [StoreOpener-8af47d28058a302c1449b0cfff8bfc3d-1 {}] regionserver.HStore(324): Store=8af47d28058a302c1449b0cfff8bfc3d/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:08,067 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,069 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34159/tmp/wal/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:30:08,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34159/tmp/wal/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:08,076 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 8af47d28058a302c1449b0cfff8bfc3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10082332320, jitterRate=-0.06100963056087494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:08,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 8af47d28058a302c1449b0cfff8bfc3d: 2023-11-19T01:30:08,077 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d., pid=6, masterSystemTime=1700357408054 2023-11-19T01:30:08,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:08,081 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:30:08,081 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8af47d28058a302c1449b0cfff8bfc3d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:30:08,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-19T01:30:08,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8af47d28058a302c1449b0cfff8bfc3d, server=jenkins-hbase6.apache.org,33353,1700357404870 in 183 msec 2023-11-19T01:30:08,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-19T01:30:08,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8af47d28058a302c1449b0cfff8bfc3d, ASSIGN in 345 msec 2023-11-19T01:30:08,092 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:08,092 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357408092"}]},"ts":"1700357408092"} 2023-11-19T01:30:08,095 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-19T01:30:08,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:08,100 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/namespace 2023-11-19T01:30:08,102 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/2/namespace 2023-11-19T01:30:08,102 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:08,102 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:30:08,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 801 msec 2023-11-19T01:30:08,107 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-19T01:30:08,118 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-11-19T01:30:08,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 15 msec 2023-11-19T01:30:08,130 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-19T01:30:08,141 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/namespace 2023-11-19T01:30:08,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-11-19T01:30:08,164 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/default 2023-11-19T01:30:08,166 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/2/namespace/hbase 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 3.221sec 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,42885,1700357404683-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:30:08,166 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,42885,1700357404683-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-19T01:30:08,169 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-19T01:30:08,169 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-19T01:30:08,233 DEBUG [Listener at localhost/34903 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4f40c0b3 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:08,238 DEBUG [Listener at localhost/34903 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b63da07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:08,240 DEBUG [hconnection-0x77657eb-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:08,242 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52998, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:08,244 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:30:08,244 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/test.cache.data in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:08,245 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:08,246 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:30:08,246 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:30:08,246 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:30:08,246 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:08,246 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:30:08,246 INFO [Listener at localhost/34903 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-19T01:30:09,865 WARN [Thread-466 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:10,228 INFO [Thread-466 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:10,234 INFO [Thread-466 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_41179_jobhistory____g1e8pp/webapp 2023-11-19T01:30:10,394 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-19T01:30:10,395 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:10,395 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:10,400 INFO [Thread-466 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:10,501 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:10,624 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:10,883 INFO [Thread-466 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-19T01:30:10,913 INFO [Thread-466 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:41179 2023-11-19T01:30:11,300 WARN [Listener at jenkins-hbase6.apache.org/40557 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:11,381 WARN [Listener at jenkins-hbase6.apache.org/40557 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:11,382 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:11,387 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_44085_cluster____.9of7dz/webapp 2023-11-19T01:30:11,458 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:11,458 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-19T01:30:11,458 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:11,458 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:11,466 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:11,675 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:11,783 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:11,790 INFO [Listener at jenkins-hbase6.apache.org/40557 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:44085 2023-11-19T01:30:12,153 WARN [Listener at jenkins-hbase6.apache.org/32815 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:12,178 WARN [Listener at jenkins-hbase6.apache.org/32815 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:12,179 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:12,188 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_45931_node____.zgtob2/webapp 2023-11-19T01:30:12,242 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:30:12,242 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:12,243 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:12,243 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:12,270 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:12,293 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:12,375 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:12,388 INFO [Listener at jenkins-hbase6.apache.org/32815 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:45931 2023-11-19T01:30:12,541 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:12,548 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:12,548 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:12,555 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_34265_node____.p2jfo4/webapp 2023-11-19T01:30:12,606 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:30:12,607 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:12,607 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:12,607 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:12,614 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:12,631 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:12,698 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:12,708 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:34265 2023-11-19T01:30:12,772 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:12,787 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-19T01:30:12,795 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:38223/backupUT 2023-11-19T01:30:12,826 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] backup.TestBackupBase(306): REMOTE ROOTDIR hdfs://localhost:34159/backupUT 2023-11-19T01:30:12,844 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:12,869 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56762, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:12,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns1'} 2023-11-19T01:30:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-19T01:30:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:30:12,939 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:12,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 40 msec 2023-11-19T01:30:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:30:13,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns2'} 2023-11-19T01:30:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-19T01:30:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:30:13,049 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:13,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 20 msec 2023-11-19T01:30:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:30:13,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns3'} 2023-11-19T01:30:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-19T01:30:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:30:13,163 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:13,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 22 msec 2023-11-19T01:30:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:30:13,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns4'} 2023-11-19T01:30:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-19T01:30:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:30:13,269 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:13,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 18 msec 2023-11-19T01:30:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:30:13,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'test-1700357412826', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1700357412826 2023-11-19T01:30:13,383 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1700357412826 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:13,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "test-1700357412826" procId is: 13 2023-11-19T01:30:13,384 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:13,385 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1700357412826 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:30:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:30:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:30:13,811 INFO [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3808ea9229abbf0011d0e0727029fc0a, NAME => 'test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1700357412826', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:30:14,224 DEBUG [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(888): Instantiated test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:14,224 DEBUG [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1668): Closing 3808ea9229abbf0011d0e0727029fc0a, disabling compactions & flushes 2023-11-19T01:30:14,224 INFO [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1690): Closing region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,224 DEBUG [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,224 DEBUG [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. after waiting 0 ms 2023-11-19T01:30:14,224 DEBUG [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,225 INFO [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1909): Closed test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,225 DEBUG [RegionOpenAndInit-test-1700357412826-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3808ea9229abbf0011d0e0727029fc0a: 2023-11-19T01:30:14,226 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1700357412826 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:14,227 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1700357414226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357414226"}]},"ts":"1700357414226"} 2023-11-19T01:30:14,229 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:14,230 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1700357412826 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:14,231 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357412826","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357414230"}]},"ts":"1700357414230"} 2023-11-19T01:30:14,232 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357412826, state=ENABLING in hbase:meta 2023-11-19T01:30:14,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357412826, region=3808ea9229abbf0011d0e0727029fc0a, ASSIGN}] 2023-11-19T01:30:14,240 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357412826, region=3808ea9229abbf0011d0e0727029fc0a, ASSIGN 2023-11-19T01:30:14,241 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357412826, region=3808ea9229abbf0011d0e0727029fc0a, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:14,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=3808ea9229abbf0011d0e0727029fc0a, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:14,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 3808ea9229abbf0011d0e0727029fc0a, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:30:14,548 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:14,559 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 3808ea9229abbf0011d0e0727029fc0a, NAME => 'test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:14,560 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:14,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357412826 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:14,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,564 INFO [StoreOpener-3808ea9229abbf0011d0e0727029fc0a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,566 INFO [StoreOpener-3808ea9229abbf0011d0e0727029fc0a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3808ea9229abbf0011d0e0727029fc0a columnFamilyName f 2023-11-19T01:30:14,566 DEBUG [StoreOpener-3808ea9229abbf0011d0e0727029fc0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:14,566 INFO [StoreOpener-3808ea9229abbf0011d0e0727029fc0a-1 {}] regionserver.HStore(324): Store=3808ea9229abbf0011d0e0727029fc0a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:14,567 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,568 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,571 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:14,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:14,575 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 3808ea9229abbf0011d0e0727029fc0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11144864000, jitterRate=0.037946343421936035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:14,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 3808ea9229abbf0011d0e0727029fc0a: 2023-11-19T01:30:14,577 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a., pid=15, masterSystemTime=1700357414548 2023-11-19T01:30:14,579 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,579 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:14,580 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=3808ea9229abbf0011d0e0727029fc0a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:14,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-19T01:30:14,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 3808ea9229abbf0011d0e0727029fc0a, server=jenkins-hbase6.apache.org,41675,1700357398894 in 187 msec 2023-11-19T01:30:14,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-19T01:30:14,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1700357412826, region=3808ea9229abbf0011d0e0727029fc0a, ASSIGN in 348 msec 2023-11-19T01:30:14,588 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1700357412826 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:14,589 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357412826","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357414588"}]},"ts":"1700357414588"} 2023-11-19T01:30:14,590 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357412826, state=ENABLED in hbase:meta 2023-11-19T01:30:14,593 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1700357412826 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:14,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1700357412826 in 1.2140 sec 2023-11-19T01:30:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:30:15,496 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1700357412826, procId: 13 completed 2023-11-19T01:30:15,498 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4941a25e to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:15,504 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a24ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:15,506 DEBUG [hconnection-0x6efc0c3a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:15,510 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33274, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:15,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HRegion(8210): writing data to region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:30:15,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns2:test-17003574128261', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-17003574128261 2023-11-19T01:30:15,620 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003574128261 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:15,620 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:15,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns2" qualifier: "test-17003574128261" procId is: 16 2023-11-19T01:30:15,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-17003574128261 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:30:15,644 INFO [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8673ecec9222769a503bdb3d24ac32cc, NAME => 'ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17003574128261', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:30:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:30:16,062 DEBUG [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:16,062 DEBUG [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1668): Closing 8673ecec9222769a503bdb3d24ac32cc, disabling compactions & flushes 2023-11-19T01:30:16,062 INFO [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,062 DEBUG [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,063 DEBUG [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. after waiting 1 ms 2023-11-19T01:30:16,063 DEBUG [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,063 INFO [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,063 DEBUG [RegionOpenAndInit-ns2:test-17003574128261-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8673ecec9222769a503bdb3d24ac32cc: 2023-11-19T01:30:16,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-17003574128261 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:16,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357416064"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357416064"}]},"ts":"1700357416064"} 2023-11-19T01:30:16,067 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:16,068 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-17003574128261 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:16,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003574128261","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357416068"}]},"ts":"1700357416068"} 2023-11-19T01:30:16,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003574128261, state=ENABLING in hbase:meta 2023-11-19T01:30:16,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003574128261, region=8673ecec9222769a503bdb3d24ac32cc, ASSIGN}] 2023-11-19T01:30:16,078 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003574128261, region=8673ecec9222769a503bdb3d24ac32cc, ASSIGN 2023-11-19T01:30:16,079 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-17003574128261, region=8673ecec9222769a503bdb3d24ac32cc, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:30:16,231 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=8673ecec9222769a503bdb3d24ac32cc, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:16,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 8673ecec9222769a503bdb3d24ac32cc, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:16,385 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:16,390 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,390 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 8673ecec9222769a503bdb3d24ac32cc, NAME => 'ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:16,391 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:16,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003574128261 8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:16,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,393 INFO [StoreOpener-8673ecec9222769a503bdb3d24ac32cc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,395 INFO [StoreOpener-8673ecec9222769a503bdb3d24ac32cc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8673ecec9222769a503bdb3d24ac32cc columnFamilyName f 2023-11-19T01:30:16,395 DEBUG [StoreOpener-8673ecec9222769a503bdb3d24ac32cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:16,396 INFO [StoreOpener-8673ecec9222769a503bdb3d24ac32cc-1 {}] regionserver.HStore(324): Store=8673ecec9222769a503bdb3d24ac32cc/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:16,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,401 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:30:16,404 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:16,404 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 8673ecec9222769a503bdb3d24ac32cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9719891840, jitterRate=-0.09476453065872192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:16,405 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 8673ecec9222769a503bdb3d24ac32cc: 2023-11-19T01:30:16,407 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc., pid=18, masterSystemTime=1700357416385 2023-11-19T01:30:16,409 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,409 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:30:16,410 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=8673ecec9222769a503bdb3d24ac32cc, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:16,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-19T01:30:16,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 8673ecec9222769a503bdb3d24ac32cc, server=jenkins-hbase6.apache.org,41675,1700357398894 in 179 msec 2023-11-19T01:30:16,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-19T01:30:16,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003574128261, region=8673ecec9222769a503bdb3d24ac32cc, ASSIGN in 340 msec 2023-11-19T01:30:16,418 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-17003574128261 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:16,419 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003574128261","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357416418"}]},"ts":"1700357416418"} 2023-11-19T01:30:16,421 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003574128261, state=ENABLED in hbase:meta 2023-11-19T01:30:16,424 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003574128261 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:16,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-17003574128261 in 808 msec 2023-11-19T01:30:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:30:16,727 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-17003574128261, procId: 16 completed 2023-11-19T01:30:16,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HRegion(8210): writing data to region ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:30:16,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns3:test-17003574128262', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-17003574128262 2023-11-19T01:30:16,828 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003574128262 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:16,828 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:16,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns3" qualifier: "test-17003574128262" procId is: 19 2023-11-19T01:30:16,829 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-17003574128262 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:30:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:30:16,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003574128261 2023-11-19T01:30:16,967 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003574128261 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:16,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1700357412826 2023-11-19T01:30:16,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1700357412826 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:30:17,244 INFO [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => dac22350d297ecf0792f58710c9ec235, NAME => 'ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17003574128262', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:30:17,660 DEBUG [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:17,660 DEBUG [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1668): Closing dac22350d297ecf0792f58710c9ec235, disabling compactions & flushes 2023-11-19T01:30:17,660 INFO [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:17,660 DEBUG [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:17,661 DEBUG [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. after waiting 0 ms 2023-11-19T01:30:17,661 DEBUG [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:17,661 INFO [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:17,661 DEBUG [RegionOpenAndInit-ns3:test-17003574128262-pool-0 {}] regionserver.HRegion(1622): Region close journal for dac22350d297ecf0792f58710c9ec235: 2023-11-19T01:30:17,662 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-17003574128262 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:17,663 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357417662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357417662"}]},"ts":"1700357417662"} 2023-11-19T01:30:17,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:17,666 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-17003574128262 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:17,666 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003574128262","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357417666"}]},"ts":"1700357417666"} 2023-11-19T01:30:17,668 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003574128262, state=ENABLING in hbase:meta 2023-11-19T01:30:17,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003574128262, region=dac22350d297ecf0792f58710c9ec235, ASSIGN}] 2023-11-19T01:30:17,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003574128262, region=dac22350d297ecf0792f58710c9ec235, ASSIGN 2023-11-19T01:30:17,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-17003574128262, region=dac22350d297ecf0792f58710c9ec235, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:17,831 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=dac22350d297ecf0792f58710c9ec235, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:17,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure dac22350d297ecf0792f58710c9ec235, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:30:17,986 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:17,991 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:17,991 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => dac22350d297ecf0792f58710c9ec235, NAME => 'ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:17,992 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:17,992 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003574128262 dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:17,992 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:17,992 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:17,992 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:17,994 INFO [StoreOpener-dac22350d297ecf0792f58710c9ec235-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:17,996 INFO [StoreOpener-dac22350d297ecf0792f58710c9ec235-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dac22350d297ecf0792f58710c9ec235 columnFamilyName f 2023-11-19T01:30:17,996 DEBUG [StoreOpener-dac22350d297ecf0792f58710c9ec235-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:17,997 INFO [StoreOpener-dac22350d297ecf0792f58710c9ec235-1 {}] regionserver.HStore(324): Store=dac22350d297ecf0792f58710c9ec235/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:17,998 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns3/test-17003574128262/dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:17,998 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns3/test-17003574128262/dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:18,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for dac22350d297ecf0792f58710c9ec235 2023-11-19T01:30:18,005 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns3/test-17003574128262/dac22350d297ecf0792f58710c9ec235/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:18,006 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened dac22350d297ecf0792f58710c9ec235; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10706612320, jitterRate=-0.002869024872779846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:18,007 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for dac22350d297ecf0792f58710c9ec235: 2023-11-19T01:30:18,008 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235., pid=21, masterSystemTime=1700357417986 2023-11-19T01:30:18,011 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:18,011 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:30:18,013 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=dac22350d297ecf0792f58710c9ec235, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:18,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-19T01:30:18,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure dac22350d297ecf0792f58710c9ec235, server=jenkins-hbase6.apache.org,41675,1700357398894 in 182 msec 2023-11-19T01:30:18,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-19T01:30:18,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003574128262, region=dac22350d297ecf0792f58710c9ec235, ASSIGN in 345 msec 2023-11-19T01:30:18,023 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-17003574128262 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:18,023 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003574128262","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357418023"}]},"ts":"1700357418023"} 2023-11-19T01:30:18,025 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003574128262, state=ENABLED in hbase:meta 2023-11-19T01:30:18,031 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003574128262 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:18,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-17003574128262 in 1.2070 sec 2023-11-19T01:30:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:30:18,938 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-17003574128262, procId: 19 completed 2023-11-19T01:30:18,939 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-17003574128262 get assigned. Timeout = 60000ms 2023-11-19T01:30:18,941 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:30:18,946 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-17003574128262 assigned to meta. Checking AM states. 2023-11-19T01:30:18,947 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:30:18,947 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-17003574128262 assigned. 2023-11-19T01:30:18,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns4:test-17003574128263', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-17003574128263 2023-11-19T01:30:18,954 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003574128263 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:18,954 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:18,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns4" qualifier: "test-17003574128263" procId is: 22 2023-11-19T01:30:18,955 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-17003574128263 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:30:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:30:19,115 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17003574128261' 2023-11-19T01:30:19,116 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1700357412826' 2023-11-19T01:30:19,116 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17003574128262' 2023-11-19T01:30:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:30:19,372 INFO [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3de69d369d7dacc90031c653c9918ad3, NAME => 'ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17003574128263', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:19,385 DEBUG [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:19,385 DEBUG [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1668): Closing 3de69d369d7dacc90031c653c9918ad3, disabling compactions & flushes 2023-11-19T01:30:19,385 INFO [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,385 DEBUG [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,385 DEBUG [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. after waiting 0 ms 2023-11-19T01:30:19,385 DEBUG [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,385 INFO [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,385 DEBUG [RegionOpenAndInit-ns4:test-17003574128263-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3de69d369d7dacc90031c653c9918ad3: 2023-11-19T01:30:19,386 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-17003574128263 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:19,387 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357419386"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357419386"}]},"ts":"1700357419386"} 2023-11-19T01:30:19,389 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:19,390 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-17003574128263 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:19,391 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003574128263","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357419390"}]},"ts":"1700357419390"} 2023-11-19T01:30:19,392 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003574128263, state=ENABLING in hbase:meta 2023-11-19T01:30:19,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003574128263, region=3de69d369d7dacc90031c653c9918ad3, ASSIGN}] 2023-11-19T01:30:19,398 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003574128263, region=3de69d369d7dacc90031c653c9918ad3, ASSIGN 2023-11-19T01:30:19,399 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-17003574128263, region=3de69d369d7dacc90031c653c9918ad3, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:19,551 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=3de69d369d7dacc90031c653c9918ad3, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:19,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 3de69d369d7dacc90031c653c9918ad3, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:30:19,710 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:19,715 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,715 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 3de69d369d7dacc90031c653c9918ad3, NAME => 'ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:19,715 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:19,716 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003574128263 3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,716 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:19,716 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,716 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,717 INFO [StoreOpener-3de69d369d7dacc90031c653c9918ad3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,719 INFO [StoreOpener-3de69d369d7dacc90031c653c9918ad3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3de69d369d7dacc90031c653c9918ad3 columnFamilyName f 2023-11-19T01:30:19,719 DEBUG [StoreOpener-3de69d369d7dacc90031c653c9918ad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:19,720 INFO [StoreOpener-3de69d369d7dacc90031c653c9918ad3-1 {}] regionserver.HStore(324): Store=3de69d369d7dacc90031c653c9918ad3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:19,721 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns4/test-17003574128263/3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,721 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns4/test-17003574128263/3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,728 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:30:19,734 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns4/test-17003574128263/3de69d369d7dacc90031c653c9918ad3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:19,735 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 3de69d369d7dacc90031c653c9918ad3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11972963840, jitterRate=0.11506915092468262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:19,736 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 3de69d369d7dacc90031c653c9918ad3: 2023-11-19T01:30:19,738 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3., pid=24, masterSystemTime=1700357419710 2023-11-19T01:30:19,740 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,740 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:30:19,741 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=3de69d369d7dacc90031c653c9918ad3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:19,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-19T01:30:19,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 3de69d369d7dacc90031c653c9918ad3, server=jenkins-hbase6.apache.org,41675,1700357398894 in 186 msec 2023-11-19T01:30:19,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-19T01:30:19,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-17003574128263, region=3de69d369d7dacc90031c653c9918ad3, ASSIGN in 351 msec 2023-11-19T01:30:19,749 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-17003574128263 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:19,749 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003574128263","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357419749"}]},"ts":"1700357419749"} 2023-11-19T01:30:19,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003574128263, state=ENABLED in hbase:meta 2023-11-19T01:30:19,755 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003574128263 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:19,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-17003574128263 in 806 msec 2023-11-19T01:30:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:30:20,064 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-17003574128263, procId: 22 completed 2023-11-19T01:30:20,064 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-17003574128263 get assigned. Timeout = 60000ms 2023-11-19T01:30:20,064 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:30:20,064 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:20,069 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-17003574128263 assigned to meta. Checking AM states. 2023-11-19T01:30:20,069 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:30:20,070 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-17003574128263 assigned. 2023-11-19T01:30:20,070 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4941a25e to 127.0.0.1:53145 2023-11-19T01:30:20,070 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:20,147 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemote Thread=817, OpenFileDescriptor=907, MaxFileDescriptor=60000, SystemLoadAverage=143, ProcessCount=166, AvailableMemoryMB=4556 2023-11-19T01:30:20,147 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.ResourceChecker(130): Thread=817 is superior to 500 2023-11-19T01:30:20,149 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] backup.TestRemoteRestore(70): test remote full backup on a single table 2023-11-19T01:30:20,152 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5c68746d to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:20,168 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f9782cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:20,215 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:20,225 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33280, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:20,251 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:20,253 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:20,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'backup'} 2023-11-19T01:30:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-19T01:30:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:30:20,283 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:30:20,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 16 msec 2023-11-19T01:30:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:30:20,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-19T01:30:20,403 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:20,403 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:20,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-19T01:30:20,405 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:30:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:30:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:30:20,828 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7234af1d9c333fce0f1fb04662aacac5, NAME => 'backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:30:21,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:21,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 7234af1d9c333fce0f1fb04662aacac5, disabling compactions & flushes 2023-11-19T01:30:21,273 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. after waiting 0 ms 2023-11-19T01:30:21,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,273 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7234af1d9c333fce0f1fb04662aacac5: 2023-11-19T01:30:21,275 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:21,275 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357421275"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357421275"}]},"ts":"1700357421275"} 2023-11-19T01:30:21,278 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:21,280 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:21,280 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357421280"}]},"ts":"1700357421280"} 2023-11-19T01:30:21,282 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:30:21,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7234af1d9c333fce0f1fb04662aacac5, ASSIGN}] 2023-11-19T01:30:21,288 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7234af1d9c333fce0f1fb04662aacac5, ASSIGN 2023-11-19T01:30:21,289 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=7234af1d9c333fce0f1fb04662aacac5, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:21,442 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7234af1d9c333fce0f1fb04662aacac5, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:21,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 7234af1d9c333fce0f1fb04662aacac5, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:30:21,596 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:21,600 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 7234af1d9c333fce0f1fb04662aacac5, NAME => 'backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:21,601 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:21,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:21,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,603 INFO [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,605 INFO [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7234af1d9c333fce0f1fb04662aacac5 columnFamilyName meta 2023-11-19T01:30:21,605 DEBUG [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:21,606 INFO [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] regionserver.HStore(324): Store=7234af1d9c333fce0f1fb04662aacac5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:21,606 INFO [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,608 INFO [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7234af1d9c333fce0f1fb04662aacac5 columnFamilyName session 2023-11-19T01:30:21,608 DEBUG [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:21,608 INFO [StoreOpener-7234af1d9c333fce0f1fb04662aacac5-1 {}] regionserver.HStore(324): Store=7234af1d9c333fce0f1fb04662aacac5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:21,609 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:30:21,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:21,616 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:21,616 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 7234af1d9c333fce0f1fb04662aacac5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9655520320, jitterRate=-0.1007595956325531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:30:21,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 7234af1d9c333fce0f1fb04662aacac5: 2023-11-19T01:30:21,618 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5., pid=28, masterSystemTime=1700357421596 2023-11-19T01:30:21,620 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,620 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:21,621 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7234af1d9c333fce0f1fb04662aacac5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:21,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-19T01:30:21,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 7234af1d9c333fce0f1fb04662aacac5, server=jenkins-hbase6.apache.org,41675,1700357398894 in 179 msec 2023-11-19T01:30:21,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-19T01:30:21,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=7234af1d9c333fce0f1fb04662aacac5, ASSIGN in 340 msec 2023-11-19T01:30:21,628 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:21,628 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357421628"}]},"ts":"1700357421628"} 2023-11-19T01:30:21,630 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:30:21,632 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:21,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.2360 sec 2023-11-19T01:30:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:30:22,513 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-19T01:30:22,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-19T01:30:22,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:22,531 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:22,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-19T01:30:22,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:30:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:30:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:30:22,945 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fb436e957cd5ee14b630b03f76f4c9d3, NAME => 'backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:30:23,357 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:23,357 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing fb436e957cd5ee14b630b03f76f4c9d3, disabling compactions & flushes 2023-11-19T01:30:23,357 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,357 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,358 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. after waiting 0 ms 2023-11-19T01:30:23,358 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,358 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,358 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for fb436e957cd5ee14b630b03f76f4c9d3: 2023-11-19T01:30:23,359 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:23,360 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1700357423359"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357423359"}]},"ts":"1700357423359"} 2023-11-19T01:30:23,362 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:23,363 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:23,363 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357423363"}]},"ts":"1700357423363"} 2023-11-19T01:30:23,365 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-19T01:30:23,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=fb436e957cd5ee14b630b03f76f4c9d3, ASSIGN}] 2023-11-19T01:30:23,373 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=fb436e957cd5ee14b630b03f76f4c9d3, ASSIGN 2023-11-19T01:30:23,376 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=fb436e957cd5ee14b630b03f76f4c9d3, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:23,528 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=fb436e957cd5ee14b630b03f76f4c9d3, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:23,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure fb436e957cd5ee14b630b03f76f4c9d3, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:30:23,683 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:23,688 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => fb436e957cd5ee14b630b03f76f4c9d3, NAME => 'backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:23,688 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:23,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:23,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,690 INFO [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,692 INFO [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb436e957cd5ee14b630b03f76f4c9d3 columnFamilyName meta 2023-11-19T01:30:23,692 DEBUG [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:23,693 INFO [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] regionserver.HStore(324): Store=fb436e957cd5ee14b630b03f76f4c9d3/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:23,693 INFO [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,694 INFO [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb436e957cd5ee14b630b03f76f4c9d3 columnFamilyName session 2023-11-19T01:30:23,694 DEBUG [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:23,695 INFO [StoreOpener-fb436e957cd5ee14b630b03f76f4c9d3-1 {}] regionserver.HStore(324): Store=fb436e957cd5ee14b630b03f76f4c9d3/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:23,696 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system_bulk/fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,696 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system_bulk/fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:30:23,700 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:30:23,703 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system_bulk/fb436e957cd5ee14b630b03f76f4c9d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:23,704 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened fb436e957cd5ee14b630b03f76f4c9d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10126907040, jitterRate=-0.05685828626155853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:30:23,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for fb436e957cd5ee14b630b03f76f4c9d3: 2023-11-19T01:30:23,705 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3., pid=31, masterSystemTime=1700357423683 2023-11-19T01:30:23,707 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,707 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:30:23,708 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=fb436e957cd5ee14b630b03f76f4c9d3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:23,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-19T01:30:23,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure fb436e957cd5ee14b630b03f76f4c9d3, server=jenkins-hbase6.apache.org,41675,1700357398894 in 178 msec 2023-11-19T01:30:23,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-19T01:30:23,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=fb436e957cd5ee14b630b03f76f4c9d3, ASSIGN in 342 msec 2023-11-19T01:30:23,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:23,715 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357423715"}]},"ts":"1700357423715"} 2023-11-19T01:30:23,717 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-19T01:30:23,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:23,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1930 sec 2023-11-19T01:30:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:30:24,640 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-19T01:30:24,654 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:30:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-19T01:30:24,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357424691 (current time:1700357424691). 2023-11-19T01:30:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:30:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:30:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x375012bd to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa2291d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:24,704 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:34138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x375012bd to 127.0.0.1:53145 2023-11-19T01:30:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:30:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-19T01:30:24,723 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:30:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:30:24,729 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:30:24,743 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot, current working dir path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:30:24,745 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:30:24,758 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:30:24,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5}] 2023-11-19T01:30:24,771 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:30:24,930 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:24,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41675 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-19T01:30:24,933 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:24,936 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 7234af1d9c333fce0f1fb04662aacac5 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-19T01:30:25,025 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/262302e0779b4b34af13b9becf90d14b is 49, key is activesession:/session:c/1700357424676/Put/seqid=0 2023-11-19T01:30:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:30:25,218 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:25,326 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-19T01:30:25,326 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17003574128263' 2023-11-19T01:30:25,327 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:30:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:30:25,454 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/262302e0779b4b34af13b9becf90d14b 2023-11-19T01:30:25,531 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/262302e0779b4b34af13b9becf90d14b as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/262302e0779b4b34af13b9becf90d14b 2023-11-19T01:30:25,539 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/262302e0779b4b34af13b9becf90d14b, entries=1, sequenceid=5, filesize=4.9 K 2023-11-19T01:30:25,544 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 7234af1d9c333fce0f1fb04662aacac5 in 608ms, sequenceid=5, compaction requested=false 2023-11-19T01:30:25,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 7234af1d9c333fce0f1fb04662aacac5: 2023-11-19T01:30:25,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. for snapshot_backup_system completed. 2023-11-19T01:30:25,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:30:25,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:30:25,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:30:25,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/262302e0779b4b34af13b9becf90d14b] hfiles 2023-11-19T01:30:25,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/262302e0779b4b34af13b9becf90d14b for snapshot=snapshot_backup_system 2023-11-19T01:30:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:30:25,977 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:25,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-19T01:30:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-19T01:30:25,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:25,986 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:25,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-19T01:30:25,989 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:30:25,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5 in 1.2250 sec 2023-11-19T01:30:25,991 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:30:25,994 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:30:25,994 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:30:25,997 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:30:26,425 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:30:26,438 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:30:26,439 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:30:26,441 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:30:26,441 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-19T01:30:26,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7260 sec 2023-11-19T01:30:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:30:26,831 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-11-19T01:30:26,832 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(115): Backup backup_1700357420173 started at 1700357426831. 2023-11-19T01:30:26,863 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(119): Backup session backup_1700357420173 has been started. 2023-11-19T01:30:26,867 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:30:26,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:30:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:30:26,884 INFO [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:30:26,884 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:30:26,886 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:30:26,886 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:30:26,887 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:26,887 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:26,891 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:30:26,891 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,891 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:30:26,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:26,892 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,892 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:30:26,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:26,893 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:26,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:30:26,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:26,893 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:30:26,895 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:30:26,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:30:26,898 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:30:26,898 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:30:26,900 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:30:26,900 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:30:26,900 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,41675,1700357398894' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:30:26,902 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,902 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:26,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:26,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:26,902 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-19T01:30:26,902 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:30:26,903 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:26,903 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:26,903 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:26,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:26,905 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,41675,1700357398894' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:30:26,905 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1d57cce5[Count = 0] remaining members to acquire global barrier 2023-11-19T01:30:26,905 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:30:26,905 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:26,907 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:26,907 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:26,907 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:26,907 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:30:26,907 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,907 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:30:26,908 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:30:26,908 DEBUG [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,909 INFO [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357401062 highest: 1700357401062 on jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,909 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C41675%2C1700357398894:(num 1700357401062) roll requested 2023-11-19T01:30:26,921 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357426909, exclude list is [], retry=0 2023-11-19T01:30:26,931 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:30:26,936 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357401062 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357426909 2023-11-19T01:30:26,938 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:30:26,938 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357401062 is not closed yet, will try archiving it next time 2023-11-19T01:30:26,938 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta:.meta(num 1700357401413) roll requested 2023-11-19T01:30:26,951 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357426938.meta, exclude list is [], retry=0 2023-11-19T01:30:26,956 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:30:26,959 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357401413.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357426938.meta 2023-11-19T01:30:26,959 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:30:26,959 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357401413.meta is not closed yet, will try archiving it next time 2023-11-19T01:30:26,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:30:26,967 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:26,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003574128263 2023-11-19T01:30:26,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003574128263 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:26,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-19T01:30:26,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:26,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003574128262 2023-11-19T01:30:26,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003574128262 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:26,970 DEBUG [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-11-19T01:30:26,970 INFO [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357426909 on jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,970 DEBUG [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:26,973 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43052, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:30:26,996 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:30:26,996 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,41675,1700357398894' in zk 2023-11-19T01:30:26,998 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:26,998 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:30:26,998 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:30:26,998 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:30:27,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:27,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:27,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:27,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:27,002 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:27,002 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:27,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:27,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,41675,1700357398894': 2023-11-19T01:30:27,005 INFO [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:30:27,005 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,41675,1700357398894' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:30:27,005 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:30:27,005 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:30:27,005 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:30:27,005 INFO [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:30:27,007 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,007 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,007 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:30:27,007 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,007 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,007 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:27,007 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,007 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:27,008 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:30:27,008 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:27,008 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:27,008 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,008 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,008 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:27,009 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:27,009 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:27,010 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,010 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:27,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:27,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:27,024 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:30:27,024 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:27,024 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:30:27,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:30:27,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:30:27,027 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:30:27,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:27,029 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-19T01:30:27,029 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-19T01:30:27,130 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-19T01:30:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:30:27,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } 2023-11-19T01:30:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357427144 (current time:1700357427144). 2023-11-19T01:30:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:30:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357427143_default_test-1700357412826 VERSION not specified, setting to 2 2023-11-19T01:30:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5675ffaa to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51378ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:27,165 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:34154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5675ffaa to 127.0.0.1:53145 2023-11-19T01:30:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:30:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } 2023-11-19T01:30:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-19T01:30:27,171 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:30:27,174 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:30:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:30:27,178 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot, current working dir path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_1700357427143_default_test-1700357412826 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:30:27,180 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:30:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:30:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:30:27,598 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:30:27,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a}] 2023-11-19T01:30:27,600 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:27,752 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:27,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41675 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-11-19T01:30:27,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:27,755 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 3808ea9229abbf0011d0e0727029fc0a 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:30:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:30:27,787 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/.tmp/f/c56b293158ad404cb13b8c0d1fc1d4bb is 37, key is row10/f:q1/1700357415530/Put/seqid=0 2023-11-19T01:30:28,211 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/.tmp/f/c56b293158ad404cb13b8c0d1fc1d4bb 2023-11-19T01:30:28,220 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/.tmp/f/c56b293158ad404cb13b8c0d1fc1d4bb as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb 2023-11-19T01:30:28,227 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:30:28,229 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 3808ea9229abbf0011d0e0727029fc0a in 474ms, sequenceid=103, compaction requested=false 2023-11-19T01:30:28,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 3808ea9229abbf0011d0e0727029fc0a: 2023-11-19T01:30:28,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. for snapshot_1700357427143_default_test-1700357412826 completed. 2023-11-19T01:30:28,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.' region-info for snapshot=snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:28,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:30:28,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb] hfiles 2023-11-19T01:30:28,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb for snapshot=snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:30:28,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:28,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-11-19T01:30:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster(4026): Remote procedure done, pid=35 2023-11-19T01:30:28,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357427143_default_test-1700357412826 on region 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:28,642 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:28,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-19T01:30:28,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a in 1.0450 sec 2023-11-19T01:30:28,645 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:30:28,646 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:30:28,647 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:30:28,647 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:28,648 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:29,064 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:30:29,071 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:30:29,071 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_1700357427143_default_test-1700357412826 to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:29,073 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:30:29,073 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-19T01:30:29,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357427143_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } in 1.9060 sec 2023-11-19T01:30:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:30:29,283 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1700357412826, procId: 34 completed 2023-11-19T01:30:29,283 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357420173 2023-11-19T01:30:29,283 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:30:29,285 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:30:29,287 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357420173_test-1700357412826 2023-11-19T01:30:29,287 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357427143_default_test-1700357412826 to hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/ 2023-11-19T01:30:29,297 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:30:29,328 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38223, inputRoot=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:29,328 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,34461,1700357395839_527236229_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/.hbase-snapshot/.tmp/snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:29,328 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38223, inputRoot=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2, snapshotDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_1700357427143_default_test-1700357412826. 2023-11-19T01:30:29,338 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_1700357427143_default_test-1700357412826 to hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/.hbase-snapshot/.tmp/snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:29,778 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:29,779 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:29,780 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:29,780 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,832 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop-8840985830135607331.jar 2023-11-19T01:30:30,833 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,833 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,834 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,834 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,835 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,835 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,836 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,836 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,836 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:30,837 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:30,838 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:30:30,838 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:30:30,839 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:30:30,839 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:30:30,840 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:30:30,840 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:30:30,841 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:30:30,841 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:30:30,842 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:30:30,842 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:30:30,843 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:30:30,847 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:30,847 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:30,848 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:30,848 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:30,848 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:30,849 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:30,849 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:30,910 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:30:30,934 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357427143_default_test-1700357412826' hfile list 2023-11-19T01:30:30,947 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-19T01:30:31,236 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir/1700357431161/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:30:31,236 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir/1700357431161/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:30:31,439 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:30:31,470 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb output=hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb 2023-11-19T01:30:31,470 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-19T01:30:32,323 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:30:32,324 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:30:32,332 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:32,332 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357427143_default_test-1700357412826 finished. 2023-11-19T01:30:32,333 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(1034): test-1700357412826 2023-11-19T01:30:32,409 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357420173 2023-11-19T01:30:32,409 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:30:32,822 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34159/backupUT/backup_1700357420173/.backup.manifest 2023-11-19T01:30:32,822 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(358): Backup backup_1700357420173 finished: type=FULL,tablelist=test-1700357412826,targetRootDir=hdfs://localhost:34159/backupUT,startts=1700357426831,completets=1700357432406,bytescopied=0 2023-11-19T01:30:32,822 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:30:32,823 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:32,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357427143_default_test-1700357412826" 2023-11-19T01:30:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357427143_default_test-1700357412826 2023-11-19T01:30:32,837 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357427143_default_test-1700357412826 for backup backup_1700357420173 succeeded. 2023-11-19T01:30:32,838 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:30:32,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:30:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:30:32,857 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:30:32,866 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:30:32,890 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(376): Backup backup_1700357420173 completed. 2023-11-19T01:30:32,890 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:32,890 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5c68746d to 127.0.0.1:53145 2023-11-19T01:30:32,890 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:32,890 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] backup.TestRemoteRestore(73): backup complete 2023-11-19T01:30:32,896 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-19T01:30:32,898 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34159/backupUT/backup_1700357420173/.backup.manifest 2023-11-19T01:30:32,902 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357420173/.backup.manifest 2023-11-19T01:30:32,910 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34159/backupUT/backup_1700357420173/.backup.manifest 2023-11-19T01:30:32,912 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357420173/.backup.manifest 2023-11-19T01:30:32,912 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.RestoreTablesClient(148): Restoring 'test-1700357412826' to 'table1' from full backup image hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826 2023-11-19T01:30:32,923 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-19T01:30:32,923 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:32,923 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f in region [hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a] 2023-11-19T01:30:32,928 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb first=row0 last=row98 2023-11-19T01:30:32,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-19T01:30:32,940 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:32,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 36 2023-11-19T01:30:32,941 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:30:32,956 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4a7c265223769d355ddd8fda3a6de8ad, NAME => 'table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:30:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:30:33,368 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:33,368 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 4a7c265223769d355ddd8fda3a6de8ad, disabling compactions & flushes 2023-11-19T01:30:33,368 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,368 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,368 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. after waiting 0 ms 2023-11-19T01:30:33,369 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,369 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,369 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4a7c265223769d355ddd8fda3a6de8ad: 2023-11-19T01:30:33,370 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:33,370 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1700357433370"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357433370"}]},"ts":"1700357433370"} 2023-11-19T01:30:33,373 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:33,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:33,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357433373"}]},"ts":"1700357433373"} 2023-11-19T01:30:33,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-19T01:30:33,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, ASSIGN}] 2023-11-19T01:30:33,382 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, ASSIGN 2023-11-19T01:30:33,383 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:33,534 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=4a7c265223769d355ddd8fda3a6de8ad, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:33,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 4a7c265223769d355ddd8fda3a6de8ad, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:30:33,688 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:33,692 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,692 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => 4a7c265223769d355ddd8fda3a6de8ad, NAME => 'table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:33,693 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:33,693 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,693 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:33,693 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,693 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,694 INFO [StoreOpener-4a7c265223769d355ddd8fda3a6de8ad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,696 INFO [StoreOpener-4a7c265223769d355ddd8fda3a6de8ad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a7c265223769d355ddd8fda3a6de8ad columnFamilyName f 2023-11-19T01:30:33,696 DEBUG [StoreOpener-4a7c265223769d355ddd8fda3a6de8ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:33,697 INFO [StoreOpener-4a7c265223769d355ddd8fda3a6de8ad-1 {}] regionserver.HStore(324): Store=4a7c265223769d355ddd8fda3a6de8ad/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:33,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,707 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:33,710 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:33,711 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened 4a7c265223769d355ddd8fda3a6de8ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12006457120, jitterRate=0.11818845570087433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:33,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for 4a7c265223769d355ddd8fda3a6de8ad: 2023-11-19T01:30:33,712 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad., pid=38, masterSystemTime=1700357433688 2023-11-19T01:30:33,715 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,715 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:33,715 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=4a7c265223769d355ddd8fda3a6de8ad, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:33,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-11-19T01:30:33,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 4a7c265223769d355ddd8fda3a6de8ad, server=jenkins-hbase6.apache.org,41675,1700357398894 in 181 msec 2023-11-19T01:30:33,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-11-19T01:30:33,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, ASSIGN in 340 msec 2023-11-19T01:30:33,722 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:33,722 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357433722"}]},"ts":"1700357433722"} 2023-11-19T01:30:33,723 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-19T01:30:33,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:33,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; CreateTableProcedure table=table1 in 790 msec 2023-11-19T01:30:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:30:34,051 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:table1, procId: 36 completed 2023-11-19T01:30:34,058 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a from hbase tables test-1700357412826 to tables table1 2023-11-19T01:30:34,059 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1700357412826 into table1 2023-11-19T01:30:34,063 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059 from hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:34,067 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x448c4ea7 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:34,074 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@413b6723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:34,075 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:34,077 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53832, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:34,095 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:30:34,095 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-19T01:30:34,101 DEBUG [hconnection-0x2ed079d6-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:34,103 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53650, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:34,106 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-19T01:30:34,107 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:30:34,107 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/partitions_592da82f-a26f-4bc0-9919-606914a3d299 2023-11-19T01:30:34,526 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:30:34,527 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53842, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:30:34,553 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:34,553 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:34,554 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:34,554 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,635 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop-4038096010303521744.jar 2023-11-19T01:30:35,636 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,636 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,637 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,638 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,638 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,639 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,640 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,640 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,641 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,642 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:35,643 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:30:35,644 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:30:35,644 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:30:35,645 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:30:35,646 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:30:35,646 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:30:35,647 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:30:35,648 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:30:35,648 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:30:35,649 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:30:35,650 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:30:35,654 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,655 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,655 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,655 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,656 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,656 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:35,657 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:35,657 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-19T01:30:35,657 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:35,657 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x448c4ea7 to 127.0.0.1:53145 2023-11-19T01:30:35,657 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:35,658 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:30:35,658 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:30:35,659 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:35,666 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:30:35,673 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:30:35,763 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir/1700357435724/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:30:35,764 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir/1700357435724/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:30:35,835 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb 2023-11-19T01:30:35,840 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:30:35,965 DEBUG [pool-323-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ea499f7 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:35,971 DEBUG [pool-323-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21288711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:35,973 DEBUG [hconnection-0x78e64f54-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:35,975 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53652, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:35,977 DEBUG [pool-323-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ea499f7 to 127.0.0.1:53145 2023-11-19T01:30:35,977 DEBUG [pool-323-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:35,978 DEBUG [pool-323-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:30:35,979 DEBUG [pool-323-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:30:35,992 DEBUG [pool-323-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059/_temporary/0/_temporary/attempt_local968022082_0002_r_000000_0/f/22fb05c971cb41a4a907d8b4017a5902 is 37, key is row10/f:q1/1700357415530/Put/seqid=0 2023-11-19T01:30:35,996 WARN [Thread-966 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:41675] Specified favored nodes: [jenkins-hbase6.apache.org:41675] 2023-11-19T01:30:36,823 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059 2023-11-19T01:30:36,825 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x14cf041e to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:36,833 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a821162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:36,834 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:36,836 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53658, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:36,845 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059/_SUCCESS 2023-11-19T01:30:36,851 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:36,854 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53844, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:36,887 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059/f/22fb05c971cb41a4a907d8b4017a5902 first=Optional[row0] last=Optional[row98] 2023-11-19T01:30:36,907 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad., hostname=jenkins-hbase6.apache.org,41675,1700357398894, seqNum=2 for row with hfile group [{f,hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059/f/22fb05c971cb41a4a907d8b4017a5902}] 2023-11-19T01:30:36,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059/f/22fb05c971cb41a4a907d8b4017a5902 for inclusion in 4a7c265223769d355ddd8fda3a6de8ad/f 2023-11-19T01:30:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:30:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:30:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HRegion(2520): Flush status journal for 4a7c265223769d355ddd8fda3a6de8ad: 2023-11-19T01:30:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/hbase-staging/bulk_output-default-table1-1700357434059/f/22fb05c971cb41a4a907d8b4017a5902 to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__jnd7plll2uarmbt9io2arcffrva1stdb39unppfl5u2tap21dsu7b3ftomu90qau/f/22fb05c971cb41a4a907d8b4017a5902 2023-11-19T01:30:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__jnd7plll2uarmbt9io2arcffrva1stdb39unppfl5u2tap21dsu7b3ftomu90qau/f/22fb05c971cb41a4a907d8b4017a5902 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f/85e483fd5e2a42acbb6524f622ba4e2b_SeqId_4_ 2023-11-19T01:30:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5632423d to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6462cd66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:36,951 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:36,957 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53672, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:36,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-19T01:30:36,967 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:36,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5632423d to 127.0.0.1:53145 2023-11-19T01:30:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:36,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__jnd7plll2uarmbt9io2arcffrva1stdb39unppfl5u2tap21dsu7b3ftomu90qau/f/22fb05c971cb41a4a907d8b4017a5902 into 4a7c265223769d355ddd8fda3a6de8ad/f as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f/85e483fd5e2a42acbb6524f622ba4e2b_SeqId_4_ - updating store file list. 2023-11-19T01:30:36,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f/85e483fd5e2a42acbb6524f622ba4e2b_SeqId_4_ into 4a7c265223769d355ddd8fda3a6de8ad/f 2023-11-19T01:30:36,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__jnd7plll2uarmbt9io2arcffrva1stdb39unppfl5u2tap21dsu7b3ftomu90qau/f/22fb05c971cb41a4a907d8b4017a5902 into 4a7c265223769d355ddd8fda3a6de8ad/f (new location: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f/85e483fd5e2a42acbb6524f622ba4e2b_SeqId_4_) 2023-11-19T01:30:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__jnd7plll2uarmbt9io2arcffrva1stdb39unppfl5u2tap21dsu7b3ftomu90qau/f/22fb05c971cb41a4a907d8b4017a5902 2023-11-19T01:30:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4b47da20 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394f9c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:37,001 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53866, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:30:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:37,006 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53686, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:30:37,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4b47da20 to 127.0.0.1:53145 2023-11-19T01:30:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:37,037 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:37,038 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x14cf041e to 127.0.0.1:53145 2023-11-19T01:30:37,038 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:37,039 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:30:37,039 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:30:37,039 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.RestoreTablesClient(233): Backup: backup_1700357420173 hdfs://localhost:34159/backupUT/backup_1700357420173/default/test-1700357412826/ 2023-11-19T01:30:37,039 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:30:37,042 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-11-19T01:30:37,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable table1 2023-11-19T01:30:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-19T01:30:37,060 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357437060"}]},"ts":"1700357437060"} 2023-11-19T01:30:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:30:37,062 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-19T01:30:37,065 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-19T01:30:37,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, UNASSIGN}] 2023-11-19T01:30:37,069 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, UNASSIGN 2023-11-19T01:30:37,069 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=4a7c265223769d355ddd8fda3a6de8ad, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:37,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; CloseRegionProcedure 4a7c265223769d355ddd8fda3a6de8ad, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:30:37,229 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:37,231 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(119): Close 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:37,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1668): Closing 4a7c265223769d355ddd8fda3a6de8ad, disabling compactions & flushes 2023-11-19T01:30:37,232 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1690): Closing region table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:37,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:37,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. after waiting 0 ms 2023-11-19T01:30:37,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:37,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:30:37,241 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:37,241 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1909): Closed table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad. 2023-11-19T01:30:37,241 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1622): Region close journal for 4a7c265223769d355ddd8fda3a6de8ad: 2023-11-19T01:30:37,243 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(163): Closed 4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:37,243 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=4a7c265223769d355ddd8fda3a6de8ad, regionState=CLOSED 2023-11-19T01:30:37,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-11-19T01:30:37,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; CloseRegionProcedure 4a7c265223769d355ddd8fda3a6de8ad, server=jenkins-hbase6.apache.org,41675,1700357398894 in 174 msec 2023-11-19T01:30:37,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-19T01:30:37,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=table1, region=4a7c265223769d355ddd8fda3a6de8ad, UNASSIGN in 180 msec 2023-11-19T01:30:37,249 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357437248"}]},"ts":"1700357437248"} 2023-11-19T01:30:37,250 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-19T01:30:37,252 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-19T01:30:37,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; DisableTableProcedure table=table1 in 201 msec 2023-11-19T01:30:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:30:37,364 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: default:table1, procId: 39 completed 2023-11-19T01:30:37,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete table1 2023-11-19T01:30:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-19T01:30:37,380 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:30:37,382 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:30:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:30:37,388 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:37,392 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f, FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/recovered.edits] 2023-11-19T01:30:37,400 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f/85e483fd5e2a42acbb6524f622ba4e2b_SeqId_4_ to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/archive/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/f/85e483fd5e2a42acbb6524f622ba4e2b_SeqId_4_ 2023-11-19T01:30:37,404 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/recovered.edits/6.seqid to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/archive/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad/recovered.edits/6.seqid 2023-11-19T01:30:37,405 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/4a7c265223769d355ddd8fda3a6de8ad 2023-11-19T01:30:37,405 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-19T01:30:37,407 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:30:37,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41675 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2023-11-19T01:30:37,418 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-19T01:30:37,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-19T01:30:37,422 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:30:37,422 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-19T01:30:37,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357437422"}]},"ts":"9223372036854775807"} 2023-11-19T01:30:37,424 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:30:37,424 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4a7c265223769d355ddd8fda3a6de8ad, NAME => 'table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:30:37,424 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-19T01:30:37,424 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357437424"}]},"ts":"9223372036854775807"} 2023-11-19T01:30:37,426 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-19T01:30:37,429 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:30:37,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; DeleteTableProcedure table=table1 in 58 msec 2023-11-19T01:30:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:30:37,484 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: DELETE, Table Name: default:table1, procId: 42 completed 2023-11-19T01:30:37,508 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemote Thread=831 (was 817) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1148165560_16 at /127.0.0.1:35954 [Receiving block BP-71530584-172.31.2.20-1700357392544:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (8930783) connection to localhost/127.0.0.1:34159 from jenkins.hfs.1 java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_178461147_16 at /127.0.0.1:57938 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1148165560_16 at /127.0.0.1:35942 [Receiving block BP-71530584-172.31.2.20-1700357392544:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-71530584-172.31.2.20-1700357392544:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-71530584-172.31.2.20-1700357392544:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_461612072_16 at /127.0.0.1:39558 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=934 (was 907) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=139 (was 143), ProcessCount=166 (was 166), AvailableMemoryMB=3785 (was 4556) 2023-11-19T01:30:37,508 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.ResourceChecker(130): Thread=831 is superior to 500 2023-11-19T01:30:37,529 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.ResourceChecker(147): before: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=831, OpenFileDescriptor=934, MaxFileDescriptor=60000, SystemLoadAverage=139, ProcessCount=166, AvailableMemoryMB=3784 2023-11-19T01:30:37,529 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.ResourceChecker(130): Thread=831 is superior to 500 2023-11-19T01:30:37,529 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] backup.TestRemoteRestore(91): test remote full backup on a single table with alternate restore output dir 2023-11-19T01:30:37,530 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1a38c359 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:37,536 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3dcce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:37,540 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:37,542 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53698, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:37,544 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:37,546 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53870, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:37,559 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:30:37,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357437565 (current time:1700357437565). 2023-11-19T01:30:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:30:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:30:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x480cdf6f to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205806c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:37,579 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53706, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x480cdf6f to 127.0.0.1:53145 2023-11-19T01:30:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:30:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-19T01:30:37,584 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:30:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:37,585 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:30:37,588 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot, current working dir path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:30:37,591 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:30:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:37,997 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:30:37,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5}] 2023-11-19T01:30:38,000 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:38,055 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:30:38,058 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55790, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:30:38,150 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:38,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41675 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-11-19T01:30:38,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:38,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 7234af1d9c333fce0f1fb04662aacac5 2/2 column families, dataSize=1.08 KB heapSize=2.23 KB 2023-11-19T01:30:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/meta/0bcb077b24e245ab98ec58093f60f398 is 169, key is trslm:hdfs://localhost:34159/backupUT\x00test-1700357412826/meta:log-roll-map/1700357432394/Put/seqid=0 2023-11-19T01:30:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:38,574 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=526 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/meta/0bcb077b24e245ab98ec58093f60f398 2023-11-19T01:30:38,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/1f82a60360864d8990596af4461cbc88 is 310, key is session:backup_1700357420173/session:context/1700357432864/Put/seqid=0 2023-11-19T01:30:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:38,988 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/1f82a60360864d8990596af4461cbc88 2023-11-19T01:30:38,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/meta/0bcb077b24e245ab98ec58093f60f398 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/meta/0bcb077b24e245ab98ec58093f60f398 2023-11-19T01:30:39,001 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/meta/0bcb077b24e245ab98ec58093f60f398, entries=4, sequenceid=17, filesize=5.5 K 2023-11-19T01:30:39,002 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/1f82a60360864d8990596af4461cbc88 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/1f82a60360864d8990596af4461cbc88 2023-11-19T01:30:39,008 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/1f82a60360864d8990596af4461cbc88, entries=2, sequenceid=17, filesize=5.3 K 2023-11-19T01:30:39,010 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.08 KB/1105, heapSize ~2.20 KB/2256, currentSize=0 B/0 for 7234af1d9c333fce0f1fb04662aacac5 in 858ms, sequenceid=17, compaction requested=false 2023-11-19T01:30:39,010 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 7234af1d9c333fce0f1fb04662aacac5: 2023-11-19T01:30:39,010 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. for snapshot_backup_system completed. 2023-11-19T01:30:39,010 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:30:39,010 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:30:39,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/meta/0bcb077b24e245ab98ec58093f60f398] hfiles 2023-11-19T01:30:39,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/meta/0bcb077b24e245ab98ec58093f60f398 for snapshot=snapshot_backup_system 2023-11-19T01:30:39,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/262302e0779b4b34af13b9becf90d14b, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/1f82a60360864d8990596af4461cbc88] hfiles 2023-11-19T01:30:39,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/262302e0779b4b34af13b9becf90d14b for snapshot=snapshot_backup_system 2023-11-19T01:30:39,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/1f82a60360864d8990596af4461cbc88 for snapshot=snapshot_backup_system 2023-11-19T01:30:39,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:39,421 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:30:39,421 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-11-19T01:30:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster(4026): Remote procedure done, pid=44 2023-11-19T01:30:39,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:39,423 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:30:39,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-11-19T01:30:39,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 7234af1d9c333fce0f1fb04662aacac5 in 1.4260 sec 2023-11-19T01:30:39,425 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:30:39,426 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:30:39,427 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:30:39,427 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:30:39,428 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:30:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:39,839 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:30:39,846 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:30:39,847 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:30:39,848 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:30:39,848 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-19T01:30:39,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2670 sec 2023-11-19T01:30:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:30:41,691 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 43 completed 2023-11-19T01:30:41,691 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(115): Backup backup_1700357437536 started at 1700357441691. 2023-11-19T01:30:41,692 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(119): Backup session backup_1700357437536 has been started. 2023-11-19T01:30:41,698 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:30:41,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:30:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:30:41,699 INFO [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:30:41,699 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:30:41,700 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:30:41,700 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:30:41,701 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,701 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:41,702 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,702 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:30:41,702 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:30:41,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:41,703 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,703 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:30:41,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:41,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,704 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:30:41,704 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:41,704 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:30:41,704 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:30:41,704 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:30:41,704 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:30:41,704 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:30:41,705 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:30:41,705 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:30:41,705 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,41675,1700357398894' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:30:41,707 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,707 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:41,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:41,707 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,707 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:30:41,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:41,708 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:41,708 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:41,708 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,709 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:41,709 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,41675,1700357398894' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:30:41,709 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5541c4b4[Count = 0] remaining members to acquire global barrier 2023-11-19T01:30:41,709 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:30:41,709 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,710 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,710 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,711 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:30:41,711 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,711 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:30:41,711 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:30:41,711 DEBUG [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,711 INFO [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357426909 highest: 1700357426909 on jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,711 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C41675%2C1700357398894:(num 1700357426909) roll requested 2023-11-19T01:30:41,722 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357441711, exclude list is [], retry=0 2023-11-19T01:30:41,726 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:30:41,728 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357426909 with entries=15, filesize=3.47 KB; new WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357441711 2023-11-19T01:30:41,728 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:30:41,728 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357426909 is not closed yet, will try archiving it next time 2023-11-19T01:30:41,728 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta:.meta(num 1700357426938) roll requested 2023-11-19T01:30:41,730 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.1700357426909 not finished, retry = 0 2023-11-19T01:30:41,740 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357441728.meta, exclude list is [], retry=0 2023-11-19T01:30:41,745 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK] 2023-11-19T01:30:41,747 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357426938.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357441728.meta 2023-11-19T01:30:41,747 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34535,DS-ae73bc2f-0ccc-4113-a7dc-fe389b3ed1ba,DISK]] 2023-11-19T01:30:41,747 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/WALs/jenkins-hbase6.apache.org,41675,1700357398894/jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta.1700357426938.meta is not closed yet, will try archiving it next time 2023-11-19T01:30:41,751 DEBUG [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:30:41,751 INFO [rs(jenkins-hbase6.apache.org,41675,1700357398894)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357441711 on jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,760 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:30:41,760 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,41675,1700357398894' in zk 2023-11-19T01:30:41,763 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:30:41,763 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,763 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:30:41,763 DEBUG [member: 'jenkins-hbase6.apache.org,41675,1700357398894' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:30:41,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:41,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:41,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:41,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:41,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:41,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:41,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:41,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,41675,1700357398894': 2023-11-19T01:30:41,767 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,41675,1700357398894' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:30:41,767 INFO [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:30:41,767 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:30:41,767 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:30:41,767 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:30:41,767 INFO [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:30:41,769 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,769 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,769 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:30:41,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,769 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:41,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:41,770 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:30:41,770 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:41,770 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:41,770 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,770 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,770 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:41,771 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:41,771 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:41,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,772 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:41,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:41,773 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,774 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,774 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:41,774 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:30:41,774 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:30:41,775 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:41,775 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:41,775 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:30:41,775 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:41,775 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:41,775 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:41,775 DEBUG [(jenkins-hbase6.apache.org,34461,1700357395839)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:30:41,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:30:41,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:30:41,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-11-19T01:30:41,776 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-19T01:30:41,776 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:30:41,776 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-19T01:30:41,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:41,876 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-19T01:30:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:30:41,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } 2023-11-19T01:30:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357441880 (current time:1700357441880). 2023-11-19T01:30:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:30:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357441879_default_test-1700357412826 VERSION not specified, setting to 2 2023-11-19T01:30:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x33677063 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9a7684, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:41,896 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:54746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x33677063 to 127.0.0.1:53145 2023-11-19T01:30:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:30:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } 2023-11-19T01:30:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-11-19T01:30:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:30:41,902 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:30:41,905 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:30:41,908 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot, current working dir path: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_1700357441879_default_test-1700357412826 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor133.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:30:41,910 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:30:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:30:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:30:42,316 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:30:42,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a}] 2023-11-19T01:30:42,319 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:42,469 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:42,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41675 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=46 2023-11-19T01:30:42,470 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.HRegion(2520): Flush status journal for 3808ea9229abbf0011d0e0727029fc0a: 2023-11-19T01:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. for snapshot_1700357441879_default_test-1700357412826 completed. 2023-11-19T01:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(240): Storing 'test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a.' region-info for snapshot=snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb] hfiles 2023-11-19T01:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb for snapshot=snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:30:42,877 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:30:42,877 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=46 2023-11-19T01:30:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster(4026): Remote procedure done, pid=46 2023-11-19T01:30:42,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357441879_default_test-1700357412826 on region 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:42,879 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:42,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-19T01:30:42,882 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:30:42,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; SnapshotRegionProcedure 3808ea9229abbf0011d0e0727029fc0a in 564 msec 2023-11-19T01:30:42,882 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:30:42,883 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:30:42,883 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:42,884 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:30:43,294 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:30:43,301 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:30:43,302 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/.tmp/snapshot_1700357441879_default_test-1700357412826 to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:43,303 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=45, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:30:43,303 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 }, snapshot procedure id = 45 2023-11-19T01:30:43,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=45, snapshot={ ss=snapshot_1700357441879_default_test-1700357412826 table=test-1700357412826 type=FLUSH ttl=0 } in 1.4050 sec 2023-11-19T01:30:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:30:44,006 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1700357412826, procId: 45 completed 2023-11-19T01:30:44,006 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357437536 2023-11-19T01:30:44,006 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:30:44,006 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:30:44,006 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357437536_test-1700357412826 2023-11-19T01:30:44,007 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357441879_default_test-1700357412826 to hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/ 2023-11-19T01:30:44,007 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:30:44,031 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38223, inputRoot=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:44,031 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,34461,1700357395839_-1410440034_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/.hbase-snapshot/.tmp/snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:44,031 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38223, inputRoot=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2, snapshotDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_1700357441879_default_test-1700357412826. 2023-11-19T01:30:44,043 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/.hbase-snapshot/snapshot_1700357441879_default_test-1700357412826 to hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/.hbase-snapshot/.tmp/snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:44,045 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-19T01:30:44,045 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-19T01:30:44,467 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:44,467 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:44,468 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:44,468 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,483 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop-3314678152349958269.jar 2023-11-19T01:30:45,483 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,483 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,484 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,485 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,485 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,485 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,486 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,486 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,486 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:45,487 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:45,488 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:30:45,488 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:30:45,489 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:30:45,489 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:30:45,490 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:30:45,490 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:30:45,490 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:30:45,491 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:30:45,491 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:30:45,492 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:30:45,492 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:30:45,493 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:45,494 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:45,494 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:45,495 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:45,495 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:30:45,496 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:45,496 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:45,524 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:30:45,545 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357441879_default_test-1700357412826' hfile list 2023-11-19T01:30:45,547 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-19T01:30:45,645 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir/1700357445604/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:30:45,645 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/mapreduce.cluster.local.dir/1700357445604/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:30:45,700 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:30:45,758 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb output=hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb 2023-11-19T01:30:45,758 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-19T01:30:46,682 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:30:46,683 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:30:46,690 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:46,690 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357441879_default_test-1700357412826 finished. 2023-11-19T01:30:46,690 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(1034): test-1700357412826 2023-11-19T01:30:46,697 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357437536 2023-11-19T01:30:46,697 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:30:46,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-19T01:30:47,102 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34159/backupUT/backup_1700357437536/.backup.manifest 2023-11-19T01:30:47,102 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(358): Backup backup_1700357437536 finished: type=FULL,tablelist=test-1700357412826,targetRootDir=hdfs://localhost:34159/backupUT,startts=1700357441691,completets=1700357446697,bytescopied=0 2023-11-19T01:30:47,102 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:30:47,103 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:47,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357441879_default_test-1700357412826" 2023-11-19T01:30:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357441879_default_test-1700357412826 2023-11-19T01:30:47,107 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357441879_default_test-1700357412826 for backup backup_1700357437536 succeeded. 2023-11-19T01:30:47,107 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:30:47,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:30:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:30:47,115 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:30:47,116 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:30:47,123 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] impl.TableBackupClient(376): Backup backup_1700357437536 completed. 2023-11-19T01:30:47,123 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:47,123 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1a38c359 to 127.0.0.1:53145 2023-11-19T01:30:47,123 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:47,123 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] backup.TestRemoteRestore(94): backup complete 2023-11-19T01:30:47,139 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir so I do NOT create it in target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef 2023-11-19T01:30:47,139 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.tmp.dir so I do NOT create it in target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef 2023-11-19T01:30:47,139 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.tmp.dir Erasing configuration value by system value. 2023-11-19T01:30:47,139 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef 2023-11-19T01:30:47,139 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea, deleteOnExit=true 2023-11-19T01:30:47,139 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/test.cache.data in system properties and HBase conf 2023-11-19T01:30:47,139 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:30:47,140 DEBUG [Listener at jenkins-hbase6.apache.org/35957 {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:47,140 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:30:47,141 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-19T01:30:47,154 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:30:47,154 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:30:47,208 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:47,210 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:47,217 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_localhost_40441_hdfs____oun1pb/webapp 2023-11-19T01:30:47,325 INFO [Listener at jenkins-hbase6.apache.org/35957 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40441 2023-11-19T01:30:47,341 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:30:47,341 WARN [Listener at jenkins-hbase6.apache.org/35957 {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:30:47,423 WARN [Listener at localhost/34767 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:47,446 WARN [Listener at localhost/34767 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:30:47,449 WARN [Listener at localhost/34767 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:47,450 INFO [Listener at localhost/34767 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:47,457 INFO [Listener at localhost/34767 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_localhost_33009_datanode____.77mh1c/webapp 2023-11-19T01:30:47,555 INFO [Listener at localhost/34767 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33009 2023-11-19T01:30:47,571 WARN [Listener at localhost/37669 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:47,589 WARN [Listener at localhost/37669 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:30:47,591 WARN [Listener at localhost/37669 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:47,592 INFO [Listener at localhost/37669 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:47,598 INFO [Listener at localhost/37669 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_localhost_39511_datanode____q7rz2u/webapp 2023-11-19T01:30:47,658 WARN [Thread-1111 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:30:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x1f82fb237ea272b4: Processing first storage report for DS-39f7e2ee-502c-42fd-a8b9-bcae6614cc73 from datanode 80cffec3-0d6f-45cc-a4dd-0c5116ecbbb7 2023-11-19T01:30:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x1f82fb237ea272b4: from storage DS-39f7e2ee-502c-42fd-a8b9-bcae6614cc73 node DatanodeRegistration(127.0.0.1:33587, datanodeUuid=80cffec3-0d6f-45cc-a4dd-0c5116ecbbb7, infoPort=44151, infoSecurePort=0, ipcPort=37669, storageInfo=lv=-57;cid=testClusterID;nsid=1536505727;c=1700357447156), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x1f82fb237ea272b4: Processing first storage report for DS-ed9c99bf-4040-4833-bd31-cc038f3a78b7 from datanode 80cffec3-0d6f-45cc-a4dd-0c5116ecbbb7 2023-11-19T01:30:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x1f82fb237ea272b4: from storage DS-ed9c99bf-4040-4833-bd31-cc038f3a78b7 node DatanodeRegistration(127.0.0.1:33587, datanodeUuid=80cffec3-0d6f-45cc-a4dd-0c5116ecbbb7, infoPort=44151, infoSecurePort=0, ipcPort=37669, storageInfo=lv=-57;cid=testClusterID;nsid=1536505727;c=1700357447156), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:47,715 INFO [Listener at localhost/37669 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39511 2023-11-19T01:30:47,752 WARN [Listener at localhost/42303 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:47,768 WARN [Listener at localhost/42303 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:30:47,771 WARN [Listener at localhost/42303 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:47,772 INFO [Listener at localhost/42303 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:47,779 INFO [Listener at localhost/42303 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_localhost_39821_datanode____lmc8u0/webapp 2023-11-19T01:30:47,821 WARN [Thread-1146 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:30:47,847 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x18a40071dc6bdb94: Processing first storage report for DS-2e27b52b-4753-424e-b30a-6339293fe225 from datanode d8387de5-a55f-4bdf-bc77-d6bc642022ba 2023-11-19T01:30:47,847 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x18a40071dc6bdb94: from storage DS-2e27b52b-4753-424e-b30a-6339293fe225 node DatanodeRegistration(127.0.0.1:42053, datanodeUuid=d8387de5-a55f-4bdf-bc77-d6bc642022ba, infoPort=43077, infoSecurePort=0, ipcPort=42303, storageInfo=lv=-57;cid=testClusterID;nsid=1536505727;c=1700357447156), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:47,847 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x18a40071dc6bdb94: Processing first storage report for DS-d158c8d8-0269-4178-964d-8ba320286f0f from datanode d8387de5-a55f-4bdf-bc77-d6bc642022ba 2023-11-19T01:30:47,848 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x18a40071dc6bdb94: from storage DS-d158c8d8-0269-4178-964d-8ba320286f0f node DatanodeRegistration(127.0.0.1:42053, datanodeUuid=d8387de5-a55f-4bdf-bc77-d6bc642022ba, infoPort=43077, infoSecurePort=0, ipcPort=42303, storageInfo=lv=-57;cid=testClusterID;nsid=1536505727;c=1700357447156), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:47,889 INFO [Listener at localhost/42303 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39821 2023-11-19T01:30:47,906 WARN [Listener at localhost/36597 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:30:47,990 WARN [Thread-1180 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:30:48,022 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x1f19154588346d62: Processing first storage report for DS-92d1bd96-df14-4eb7-a247-ca6d41868f49 from datanode 3fd6d397-3e34-4711-8053-e1fa0516b918 2023-11-19T01:30:48,022 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x1f19154588346d62: from storage DS-92d1bd96-df14-4eb7-a247-ca6d41868f49 node DatanodeRegistration(127.0.0.1:36291, datanodeUuid=3fd6d397-3e34-4711-8053-e1fa0516b918, infoPort=33491, infoSecurePort=0, ipcPort=36597, storageInfo=lv=-57;cid=testClusterID;nsid=1536505727;c=1700357447156), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:48,022 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x1f19154588346d62: Processing first storage report for DS-f1d6e84a-fbcc-4717-8a31-4e5ea4201d3f from datanode 3fd6d397-3e34-4711-8053-e1fa0516b918 2023-11-19T01:30:48,023 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x1f19154588346d62: from storage DS-f1d6e84a-fbcc-4717-8a31-4e5ea4201d3f node DatanodeRegistration(127.0.0.1:36291, datanodeUuid=3fd6d397-3e34-4711-8053-e1fa0516b918, infoPort=33491, infoSecurePort=0, ipcPort=36597, storageInfo=lv=-57;cid=testClusterID;nsid=1536505727;c=1700357447156), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:30:48,116 DEBUG [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef 2023-11-19T01:30:48,116 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-19T01:30:48,116 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/test.cache.data in system properties and HBase conf 2023-11-19T01:30:48,116 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:30:48,116 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:30:48,116 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:30:48,117 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:30:48,117 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:30:48,117 INFO [Listener at localhost/36597 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:48,118 INFO [Listener at localhost/36597 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:30:48,121 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:48,121 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:30:48,121 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:30:48,121 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:30:48,122 INFO [Listener at localhost/36597 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-19T01:30:48,237 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-19T01:30:48,238 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-19T01:30:48,640 WARN [Thread-1299 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:48,647 WARN [Thread-1299 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:48,647 INFO [Thread-1299 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:48,655 INFO [Thread-1299 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_41875_jobhistory____1y4s68/webapp 2023-11-19T01:30:48,706 INFO [Thread-1299 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-19T01:30:48,706 INFO [Thread-1299 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:48,706 INFO [Thread-1299 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:48,706 INFO [Thread-1299 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:48,713 INFO [Thread-1299 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:48,752 INFO [Thread-1299 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:48,806 INFO [Thread-1299 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-19T01:30:48,812 INFO [Thread-1299 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:41875 2023-11-19T01:30:50,131 WARN [Listener at jenkins-hbase6.apache.org/33841 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:50,141 WARN [Listener at jenkins-hbase6.apache.org/33841 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:50,142 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:50,149 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_37261_cluster____.d0xrib/webapp 2023-11-19T01:30:50,202 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:50,203 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-19T01:30:50,203 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:50,203 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:50,210 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,296 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,358 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,365 INFO [Listener at jenkins-hbase6.apache.org/33841 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:37261 2023-11-19T01:30:50,486 WARN [Listener at jenkins-hbase6.apache.org/46685 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:50,492 WARN [Listener at jenkins-hbase6.apache.org/46685 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:50,493 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:50,500 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_41581_node____.cewpir/webapp 2023-11-19T01:30:50,550 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:30:50,550 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:50,550 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:50,550 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:50,557 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,573 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,700 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,706 INFO [Listener at jenkins-hbase6.apache.org/46685 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:41581 2023-11-19T01:30:50,803 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:30:50,809 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:30:50,810 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:30:50,820 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_45339_node____.p9gde4/webapp 2023-11-19T01:30:50,879 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:30:50,879 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:30:50,879 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:30:50,879 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:30:50,887 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,902 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,963 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:30:50,970 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:45339 2023-11-19T01:30:50,973 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-19T01:30:50,975 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x73fe804a to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:50,982 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ff37f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:50,984 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:50,987 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:51902, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:50,989 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-19T01:30:50,990 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34159/backupUT/backup_1700357437536/.backup.manifest 2023-11-19T01:30:50,993 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357437536/.backup.manifest 2023-11-19T01:30:50,994 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34159/backupUT/backup_1700357437536/.backup.manifest 2023-11-19T01:30:50,996 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357437536/.backup.manifest 2023-11-19T01:30:50,996 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] impl.RestoreTablesClient(148): Restoring 'test-1700357412826' to 'table1' from full backup image hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826 2023-11-19T01:30:51,005 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-19T01:30:51,005 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:51,006 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f in region [hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a] 2023-11-19T01:30:51,009 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/f/c56b293158ad404cb13b8c0d1fc1d4bb first=row0 last=row98 2023-11-19T01:30:51,010 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:51,014 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33928, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:51,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-19T01:30:51,020 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:30:51,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 47 2023-11-19T01:30:51,025 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:30:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:30:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:30:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:30:51,432 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => dfa80d2106547a7e66a761ea1561f27c, NAME => 'table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2 2023-11-19T01:30:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:30:51,839 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:51,839 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing dfa80d2106547a7e66a761ea1561f27c, disabling compactions & flushes 2023-11-19T01:30:51,839 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:51,839 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:51,840 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. after waiting 0 ms 2023-11-19T01:30:51,840 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:51,840 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:51,840 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for dfa80d2106547a7e66a761ea1561f27c: 2023-11-19T01:30:51,841 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:30:51,841 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1700357451841"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357451841"}]},"ts":"1700357451841"} 2023-11-19T01:30:51,843 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:51,844 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:30:51,844 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357451844"}]},"ts":"1700357451844"} 2023-11-19T01:30:51,845 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-19T01:30:51,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, ASSIGN}] 2023-11-19T01:30:51,853 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, ASSIGN 2023-11-19T01:30:51,853 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,41675,1700357398894; forceNewPlan=false, retain=false 2023-11-19T01:30:52,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=dfa80d2106547a7e66a761ea1561f27c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:52,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure dfa80d2106547a7e66a761ea1561f27c, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:30:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:30:52,167 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:52,190 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:52,191 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => dfa80d2106547a7e66a761ea1561f27c, NAME => 'table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:52,191 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:52,191 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,191 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:52,191 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,191 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,193 INFO [StoreOpener-dfa80d2106547a7e66a761ea1561f27c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,194 INFO [StoreOpener-dfa80d2106547a7e66a761ea1561f27c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dfa80d2106547a7e66a761ea1561f27c columnFamilyName f 2023-11-19T01:30:52,195 DEBUG [StoreOpener-dfa80d2106547a7e66a761ea1561f27c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:52,195 INFO [StoreOpener-dfa80d2106547a7e66a761ea1561f27c-1 {}] regionserver.HStore(324): Store=dfa80d2106547a7e66a761ea1561f27c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:52,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,199 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:30:52,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:30:52,225 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened dfa80d2106547a7e66a761ea1561f27c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9498497440, jitterRate=-0.11538349092006683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:30:52,225 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for dfa80d2106547a7e66a761ea1561f27c: 2023-11-19T01:30:52,226 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c., pid=49, masterSystemTime=1700357452167 2023-11-19T01:30:52,228 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:52,228 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:30:52,228 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=dfa80d2106547a7e66a761ea1561f27c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:30:52,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-11-19T01:30:52,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure dfa80d2106547a7e66a761ea1561f27c, server=jenkins-hbase6.apache.org,41675,1700357398894 in 215 msec 2023-11-19T01:30:52,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-11-19T01:30:52,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, ASSIGN in 382 msec 2023-11-19T01:30:52,233 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:30:52,233 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357452233"}]},"ts":"1700357452233"} 2023-11-19T01:30:52,235 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-19T01:30:52,237 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=47, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:30:52,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; CreateTableProcedure table=table1 in 1.2220 sec 2023-11-19T01:30:52,736 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:30:53,132 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:table1, procId: 47 completed 2023-11-19T01:30:53,141 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a from hbase tables test-1700357412826 to tables table1 2023-11-19T01:30:53,141 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1700357412826 into table1 2023-11-19T01:30:53,145 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141 from hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/archive/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:30:53,146 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x68121f5b to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:30:53,155 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195bce27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:53,156 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:53,163 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33938, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:53,164 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:30:53,164 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-19T01:30:53,192 DEBUG [hconnection-0x7d6ee4c2-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:53,201 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:51916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:53,205 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-19T01:30:53,205 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:30:53,206 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to /user/jenkins/hbase-staging/partitions_bfda139c-be79-48c8-b2c3-dedafd84362a 2023-11-19T01:30:53,219 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:53,220 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:53,221 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:53,221 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,232 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/hadoop-1081013277842525170.jar 2023-11-19T01:30:54,232 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,232 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,233 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,233 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,234 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,234 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,234 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,235 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,235 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,236 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:54,237 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:30:54,237 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:30:54,237 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:30:54,238 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:30:54,238 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:30:54,239 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:30:54,239 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:30:54,239 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:30:54,240 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:30:54,240 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:30:54,241 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:30:54,241 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,242 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,242 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,242 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,243 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,243 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:54,244 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:54,244 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-19T01:30:54,244 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:30:54,244 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x68121f5b to 127.0.0.1:53145 2023-11-19T01:30:54,244 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:54,244 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:30:54,244 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:30:54,245 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:54,438 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:30:54,916 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:30:55,279 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(815): maximum-am-resource-percent is insufficient to start a single application in queue, it is likely set too low. skipping enforcement to allow at least one application to start 2023-11-19T01:30:55,279 WARN [SchedulerEventDispatcher:Event Processor {}] capacity.LeafQueue(851): maximum-am-resource-percent is insufficient to start a single application in queue for user, it is likely set too low. skipping enforcement to allow at least one application to start 2023-11-19T01:30:55,330 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-11-19T01:30:55,940 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1700357450370_0001_000001 (auth:SIMPLE) 2023-11-19T01:30:56,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-19T01:30:56,967 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:30:57,696 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:31:01,011 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1700357450370_0001_000001 (auth:SIMPLE) 2023-11-19T01:31:02,930 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 92f2c176e8984c832db821343de7a411 changed from -1.0 to 0.0, refreshing cache 2023-11-19T01:31:03,291 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1700357450370_0001_000001 (auth:SIMPLE) 2023-11-19T01:31:09,180 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1700357450370_0001_000001 (auth:SIMPLE) 2023-11-19T01:31:09,193 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 8af47d28058a302c1449b0cfff8bfc3d changed from -1.0 to 0.0, refreshing cache 2023-11-19T01:31:12,488 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44024, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:31:12,917 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1700357450370_0001_000001 (auth:SIMPLE) 2023-11-19T01:31:12,938 WARN [ContainersLauncher #1 {}] nodemanager.DefaultContainerExecutor(305): Exit code from container container_1700357450370_0001_01_000003 is : 143 2023-11-19T01:31:14,644 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141 2023-11-19T01:31:14,645 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3ff08453 to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:31:14,653 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6684de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:31:14,655 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:31:14,658 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:31:14,673 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141/_SUCCESS 2023-11-19T01:31:14,680 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:31:14,684 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43596, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:31:14,707 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141/f/89f6a81aa10e4892920f87d555535eea first=Optional[row0] last=Optional[row98] 2023-11-19T01:31:14,709 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c., hostname=jenkins-hbase6.apache.org,41675,1700357398894, seqNum=2 for row with hfile group [{f,hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141/f/89f6a81aa10e4892920f87d555535eea}] 2023-11-19T01:31:14,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141/f/89f6a81aa10e4892920f87d555535eea for inclusion in dfa80d2106547a7e66a761ea1561f27c/f 2023-11-19T01:31:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:31:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:31:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HRegion(2520): Flush status journal for dfa80d2106547a7e66a761ea1561f27c: 2023-11-19T01:31:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:38223/backupUT/bulk_output-default-table1-1700357453141/f/89f6a81aa10e4892920f87d555535eea to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__lg2nj3hkaq5u6i3t4iatq9mbi5lhlg9tel1n8ooq8nm5npr9ap0ttltt77vahu8a/f/89f6a81aa10e4892920f87d555535eea 2023-11-19T01:31:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__lg2nj3hkaq5u6i3t4iatq9mbi5lhlg9tel1n8ooq8nm5npr9ap0ttltt77vahu8a/f/89f6a81aa10e4892920f87d555535eea as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f/7b9b9e2757b04a7db794afb6cef1b035_SeqId_4_ 2023-11-19T01:31:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x016d402e to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:31:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4923db48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:31:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:31:14,746 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43606, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:31:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:31:14,752 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:31:14,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:31:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x016d402e to 127.0.0.1:53145 2023-11-19T01:31:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:14,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__lg2nj3hkaq5u6i3t4iatq9mbi5lhlg9tel1n8ooq8nm5npr9ap0ttltt77vahu8a/f/89f6a81aa10e4892920f87d555535eea into dfa80d2106547a7e66a761ea1561f27c/f as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f/7b9b9e2757b04a7db794afb6cef1b035_SeqId_4_ - updating store file list. 2023-11-19T01:31:14,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f/7b9b9e2757b04a7db794afb6cef1b035_SeqId_4_ into dfa80d2106547a7e66a761ea1561f27c/f 2023-11-19T01:31:14,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__lg2nj3hkaq5u6i3t4iatq9mbi5lhlg9tel1n8ooq8nm5npr9ap0ttltt77vahu8a/f/89f6a81aa10e4892920f87d555535eea into dfa80d2106547a7e66a761ea1561f27c/f (new location: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f/7b9b9e2757b04a7db794afb6cef1b035_SeqId_4_) 2023-11-19T01:31:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/staging/jenkins__table1__lg2nj3hkaq5u6i3t4iatq9mbi5lhlg9tel1n8ooq8nm5npr9ap0ttltt77vahu8a/f/89f6a81aa10e4892920f87d555535eea 2023-11-19T01:31:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x677653cc to 127.0.0.1:53145 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:31:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9df24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:31:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:31:14,795 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43620, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:31:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:31:14,803 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44054, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:31:14,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:31:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x677653cc to 127.0.0.1:53145 2023-11-19T01:31:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:14,832 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:31:14,833 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3ff08453 to 127.0.0.1:53145 2023-11-19T01:31:14,833 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:14,833 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:31:14,833 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:31:14,833 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] impl.RestoreTablesClient(233): Backup: backup_1700357437536 hdfs://localhost:34159/backupUT/backup_1700357437536/default/test-1700357412826/ 2023-11-19T01:31:14,833 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:31:14,841 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.HBaseAdmin$18(967): Started disable of table1 2023-11-19T01:31:14,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable table1 2023-11-19T01:31:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-19T01:31:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:31:14,847 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357474846"}]},"ts":"1700357474846"} 2023-11-19T01:31:14,848 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-19T01:31:14,849 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-19T01:31:14,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, UNASSIGN}] 2023-11-19T01:31:14,853 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, UNASSIGN 2023-11-19T01:31:14,854 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=dfa80d2106547a7e66a761ea1561f27c, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:31:14,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; CloseRegionProcedure dfa80d2106547a7e66a761ea1561f27c, server=jenkins-hbase6.apache.org,41675,1700357398894}] 2023-11-19T01:31:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:31:15,007 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:31:15,007 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(119): Close dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:31:15,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1668): Closing dfa80d2106547a7e66a761ea1561f27c, disabling compactions & flushes 2023-11-19T01:31:15,008 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1690): Closing region table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:31:15,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:31:15,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. after waiting 0 ms 2023-11-19T01:31:15,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:31:15,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:31:15,020 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:15,021 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1909): Closed table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c. 2023-11-19T01:31:15,021 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] regionserver.HRegion(1622): Region close journal for dfa80d2106547a7e66a761ea1561f27c: 2023-11-19T01:31:15,023 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=52}] handler.UnassignRegionHandler(163): Closed dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:31:15,025 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=dfa80d2106547a7e66a761ea1561f27c, regionState=CLOSED 2023-11-19T01:31:15,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-11-19T01:31:15,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; CloseRegionProcedure dfa80d2106547a7e66a761ea1561f27c, server=jenkins-hbase6.apache.org,41675,1700357398894 in 171 msec 2023-11-19T01:31:15,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-19T01:31:15,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=table1, region=dfa80d2106547a7e66a761ea1561f27c, UNASSIGN in 178 msec 2023-11-19T01:31:15,032 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357475031"}]},"ts":"1700357475031"} 2023-11-19T01:31:15,033 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-19T01:31:15,035 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-19T01:31:15,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; DisableTableProcedure table=table1 in 195 msec 2023-11-19T01:31:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:31:15,149 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: default:table1, procId: 50 completed 2023-11-19T01:31:15,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete table1 2023-11-19T01:31:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-19T01:31:15,154 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=53, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:31:15,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=53, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:31:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:31:15,157 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:31:15,159 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f, FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/recovered.edits] 2023-11-19T01:31:15,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f/7b9b9e2757b04a7db794afb6cef1b035_SeqId_4_ to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/archive/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/f/7b9b9e2757b04a7db794afb6cef1b035_SeqId_4_ 2023-11-19T01:31:15,167 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/recovered.edits/6.seqid to hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/archive/data/default/table1/dfa80d2106547a7e66a761ea1561f27c/recovered.edits/6.seqid 2023-11-19T01:31:15,167 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/table1/dfa80d2106547a7e66a761ea1561f27c 2023-11-19T01:31:15,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-19T01:31:15,170 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=53, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:31:15,175 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-19T01:31:15,177 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-19T01:31:15,179 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=53, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:31:15,179 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-19T01:31:15,179 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357475179"}]},"ts":"9223372036854775807"} 2023-11-19T01:31:15,181 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:31:15,181 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => dfa80d2106547a7e66a761ea1561f27c, NAME => 'table1,,1700357451015.dfa80d2106547a7e66a761ea1561f27c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:31:15,181 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-19T01:31:15,181 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357475181"}]},"ts":"9223372036854775807"} 2023-11-19T01:31:15,183 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-19T01:31:15,185 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=53, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:31:15,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; DeleteTableProcedure table=table1 in 35 msec 2023-11-19T01:31:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:31:15,257 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.HBaseAdmin$TableFuture(3745): Operation: DELETE, Table Name: default:table1, procId: 53 completed 2023-11-19T01:31:15,319 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.ResourceChecker(175): after: backup.TestRemoteRestore#testFullRestoreRemoteWithAlternateRestoreOutputDir Thread=1306 (was 831) Potentially hanging thread: IPC Server handler 42 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-400-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@19fe52a1 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #45 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-46 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 1 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_94865475_1 at /127.0.0.1:55902 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Thread[Thread-1300,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 294973869@qtp-459825326-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: 1532527460@qtp-1344623973-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:37261 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data4/current/BP-1552594006-172.31.2.20-1700357447156 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 44 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 204145774@qtp-334927031-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39821 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 9 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@66e6b0ae sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #35 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DeletionService #3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 46685 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 34767 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 29 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 48 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 37669 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 41613 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 7 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@485738b7 sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #49 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 37669 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data5/current/BP-1552594006-172.31.2.20-1700357447156 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 43 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data3/current/BP-1552594006-172.31.2.20-1700357447156 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@43013d62 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:3841) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 27 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 44 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-51 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 34 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 30 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #63 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-363-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread-1342 java.lang.Thread.sleep(Native Method) org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:143) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 18 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-41 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: ApplicationMasterLauncher #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 36597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Client (8930783) connection to localhost/127.0.0.1:34767 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 3 on default port 34767 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-58 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: 378273740@qtp-82355678-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: Timer-65 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #43 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 36 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-67 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 34767 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ContainersLauncher #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 19 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 586441612@qtp-170554572-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33009 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 0 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-61 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-39 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 5 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 35 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #42 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 37669 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 46685 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 23 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 38 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 42303 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 41013 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-63 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 7 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-59 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@27a3f30a java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #60 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 41613 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data2/current/BP-1552594006-172.31.2.20-1700357447156 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 42303 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 36 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-1552594006-172.31.2.20-1700357447156 heartbeating to localhost/127.0.0.1:34767 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RS-EventLoopGroup-5-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:316) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:373) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 21 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34767 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 41013 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@4c13f0b1 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53145@0x73fe804a-SendThread(127.0.0.1:53145) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: IPC Server handler 35 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 36597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-367-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 7 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-42 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 29 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 37669 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-377-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 36597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1638050594@qtp-2122886040-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 17 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 2030936970@qtp-1808850215-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40441 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server handler 3 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 47 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 34 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #52 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 33667 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 4 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #66 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-66 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Thread[Thread-1335,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: nioEventLoopGroup-12-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 41013 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 878339718@qtp-82355678-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:45339 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data6) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 30 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-64 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 49 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@387be014 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-40 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O server boss #51 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-50 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-371-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Thread-2050 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.readLine(BufferedReader.java:324) java.io.BufferedReader.readLine(BufferedReader.java:389) org.apache.hadoop.util.Shell$1.run(Shell.java:955) Potentially hanging thread: New I/O worker #58 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 6 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-57 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: New I/O worker #62 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 5 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #54 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-44 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 10 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 33841 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 18 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-54 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 48 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data2) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 7 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 34767 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 44 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 34767 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@32fc546e[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: BP-1552594006-172.31.2.20-1700357447156 heartbeating to localhost/127.0.0.1:34767 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #37 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 31 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 2 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 42303 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 46 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: BP-1552594006-172.31.2.20-1700357447156 heartbeating to localhost/127.0.0.1:34767 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:715) org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:851) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 46053 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 23 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 12 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: MutableQuantiles-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 44351 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data5) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_178461147_16 at /127.0.0.1:43306 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 38 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-356-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (8930783) connection to localhost/127.0.0.1:34767 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: Container metrics unregistration java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 17 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: PacketResponder: BP-71530584-172.31.2.20-1700357392544:blk_1073741871_1047, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-403-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 41013 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-401-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-361-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 20 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #56 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 29 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1148165560_16 at /127.0.0.1:40414 [Receiving block BP-71530584-172.31.2.20-1700357392544:blk_1073741871_1047] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 12 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #61 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 46685 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Thread[Thread-1325,5,FailOnTimeoutGroup] java.lang.Thread.sleep(Native Method) org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:694) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 46685 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1148165560_16 at /127.0.0.1:40404 [Receiving block BP-71530584-172.31.2.20-1700357392544:blk_1073741870_1046] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 32 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 37669 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 40 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 23 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 395342618@qtp-1614548651-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:41875 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: CacheReplicationMonitor(1769051643) sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181) Potentially hanging thread: New I/O worker #48 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@7bc7c44[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-62 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 26 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 33 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1838104893@qtp-170554572-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 10 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_94865475_1 at /127.0.0.1:41160 [Waiting for operation #2] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@66e6195c sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:113) org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 2 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 37919 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 40 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O server boss #68 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 47 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: nioEventLoopGroup-10-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #36 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #57 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@2f093c6 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:533) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-18 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 32 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 7 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data6/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 36 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 13 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 1647697703@qtp-459825326-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39511 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: IPC Server idle connection scanner for port 32943 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 34767 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #41 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 46685 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 27 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-45 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53145@0x73fe804a-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@280e3dc2 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-53 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 2 on default port 46685 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 41013 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-47 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 48 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #64 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@4cd8da6f java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@4d817530 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:244) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 25 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 31 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 15 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 28 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 40793 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 10 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 39 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 37 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 3 on default port 41013 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 41 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@164d26d6 java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 4 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 24 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 36597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Listener at jenkins-hbase6.apache.org/41013 java.lang.Thread.dumpThreads(Native Method) java.lang.Thread.getAllStackTraces(Thread.java:1615) org.apache.hadoop.hbase.ResourceCheckerJUnitListener$ThreadResourceAnalyzer.getVal(ResourceCheckerJUnitListener.java:49) org.apache.hadoop.hbase.ResourceChecker.fill(ResourceChecker.java:110) org.apache.hadoop.hbase.ResourceChecker.fillEndings(ResourceChecker.java:104) org.apache.hadoop.hbase.ResourceChecker.end(ResourceChecker.java:206) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.end(ResourceCheckerJUnitListener.java:165) org.apache.hadoop.hbase.ResourceCheckerJUnitListener.testFinished(ResourceCheckerJUnitListener.java:185) org.junit.runner.notification.SynchronizedRunListener.testFinished(SynchronizedRunListener.java:87) org.junit.runner.notification.RunNotifier$9.notifyListener(RunNotifier.java:225) org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) org.junit.runner.notification.RunNotifier.fireTestFinished(RunNotifier.java:222) org.junit.internal.runners.model.EachTestNotifier.fireTestFinished(EachTestNotifier.java:38) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:372) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 49 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #46 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 0 on default port 42303 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ContainersLauncher #0 java.io.FileInputStream.readBytes(Native Method) java.io.FileInputStream.read(FileInputStream.java:255) java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.io.BufferedInputStream.read(BufferedInputStream.java:345) sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284) sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326) sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178) java.io.InputStreamReader.read(InputStreamReader.java:184) java.io.BufferedReader.fill(BufferedReader.java:161) java.io.BufferedReader.read1(BufferedReader.java:212) java.io.BufferedReader.read(BufferedReader.java:286) org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1229) org.apache.hadoop.util.Shell.runCommand(Shell.java:984) org.apache.hadoop.util.Shell.run(Shell.java:884) org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1216) org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:294) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.launchContainer(ContainerLaunch.java:447) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:298) org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:99) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 28 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #38 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 8 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 19 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 20 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data5/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 16 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@151d6400 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:3883) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server idle connection scanner for port 42303 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: Timer-48 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 46 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data1/current/BP-1552594006-172.31.2.20-1700357447156 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-373-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 27 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server idle connection scanner for port 36597 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 14 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 17 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 16 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 6 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #59 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 9 on default port 40793 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ForkJoinPool-2-worker-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 41 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: DeletionService #0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #53 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 3 on default port 37669 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 74384103@qtp-1614548651-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53145@0x73fe804a sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:328) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$530/866330396.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #40 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: 210827776@qtp-1344623973-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: IPC Server handler 35 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 8 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: PacketResponder: BP-71530584-172.31.2.20-1700357392544:blk_1073741870_1046, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 42303 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data4) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 15 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data1) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 13 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 22 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: ForkJoinPool-2-worker-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Potentially hanging thread: IPC Server handler 39 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 1 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 5 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-354-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #55 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 39 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 43 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: hconnection-0x1cd2e91b-shared-pool-17 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 43 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #65 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-49 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: nioEventLoopGroup-14-1 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-405-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-56 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 0 on default port 33841 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 11 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-52 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 38 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #67 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 7 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #50 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 1 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data6/current/BP-1552594006-172.31.2.20-1700357447156 java.lang.Thread.sleep(Native Method) org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:205) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 30 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 14 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 21 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 36597 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: New I/O worker #47 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 4 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 42 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 10 on default port 46053 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 689176161@qtp-334927031-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@2b8b15cb java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:3975) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: pool-351-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@4a248a10 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:536) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 20 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 24 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 49 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@1dc8b331[State = -1, empty queue] sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer-43 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: IPC Server handler 24 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 18 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: pool-349-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Client (8930783) connection to jenkins-hbase6.apache.org/172.31.2.20:34615 from jenkins java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1053) org.apache.hadoop.ipc.Client$Connection.run(Client.java:1097) Potentially hanging thread: IPC Server handler 10 on default port 32943 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 9 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: 837805649@qtp-1808850215-0 java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Potentially hanging thread: 1955341134@qtp-2122886040-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:41581 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Potentially hanging thread: New I/O worker #39 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: New I/O worker #44 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/d81dcb4e-2a66-5d3c-cb54-e4fb520011ef/cluster_c17a280b-8065-6796-8c76-d5efc5db93ea/dfs/data/data3) java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:627) Potentially hanging thread: IPC Server handler 3 on default port 33667 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 45 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 26 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: IPC Server handler 0 on default port 44351 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-60 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.util.JvmPauseMonitor$Monitor@17a8b93f java.lang.Thread.sleep(Native Method) org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: IPC Server handler 40 on default port 37919 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:294) org.apache.hadoop.ipc.Server$Handler.run(Server.java:2820) Potentially hanging thread: Timer-55 java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: pool-357-thread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=1280 (was 934) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=197 (was 139) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 166) - ProcessCount LEAK? -, AvailableMemoryMB=2980 (was 3784) 2023-11-19T01:31:15,322 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.ResourceChecker(130): Thread=1306 is superior to 500 2023-11-19T01:31:15,322 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.ResourceChecker(130): OpenFileDescriptor=1280 is superior to 1024 2023-11-19T01:31:15,330 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-19T01:31:15,330 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4f40c0b3 to 127.0.0.1:53145 2023-11-19T01:31:15,330 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:15,337 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-19T01:31:15,337 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.JVMClusterUtil(257): Found active master hash=218231308, stopped=false 2023-11-19T01:31:15,337 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:31:15,340 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-11-19T01:31:15,340 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/running 2023-11-19T01:31:15,340 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-19T01:31:15,340 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:31:15,340 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:31:15,340 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:15,340 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,33353,1700357404870' ***** 2023-11-19T01:31:15,340 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-19T01:31:15,340 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:31:15,340 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-19T01:31:15,340 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/running 2023-11-19T01:31:15,364 INFO [RS:0;jenkins-hbase6:33353 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@80d046f{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:31:15,368 INFO [RS:0;jenkins-hbase6:33353 {}] server.AbstractConnector(383): Stopped ServerConnector@1dc53c84{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:31:15,368 INFO [RS:0;jenkins-hbase6:33353 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:31:15,370 INFO [RS:0;jenkins-hbase6:33353 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6870b91f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:31:15,372 INFO [RS:0;jenkins-hbase6:33353 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@700ecd02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop.log.dir/,STOPPED} 2023-11-19T01:31:15,374 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-19T01:31:15,374 INFO [RS:0;jenkins-hbase6:33353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-19T01:31:15,375 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-19T01:31:15,375 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-19T01:31:15,375 INFO [RS:0;jenkins-hbase6:33353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-19T01:31:15,375 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(3528): Received CLOSE for 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:31:15,377 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:31:15,377 DEBUG [RS:0;jenkins-hbase6:33353 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:15,378 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-19T01:31:15,378 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-19T01:31:15,378 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-19T01:31:15,378 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-19T01:31:15,379 INFO [regionserver/jenkins-hbase6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:31:15,379 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-11-19T01:31:15,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8af47d28058a302c1449b0cfff8bfc3d, disabling compactions & flushes 2023-11-19T01:31:15,380 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:31:15,380 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 8af47d28058a302c1449b0cfff8bfc3d=hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d.} 2023-11-19T01:31:15,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:31:15,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. after waiting 0 ms 2023-11-19T01:31:15,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:31:15,381 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:31:15,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8af47d28058a302c1449b0cfff8bfc3d 1/1 column families, dataSize=78 B heapSize=488 B 2023-11-19T01:31:15,382 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:31:15,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:31:15,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:31:15,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:31:15,385 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=1.26 KB heapSize=2.89 KB 2023-11-19T01:31:15,386 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:31:15,406 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/.tmp/info/c7ece878f671440ca2ea46332c384c14 is 143, key is hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d./info:regioninfo/1700357408081/Put/seqid=0 2023-11-19T01:31:15,407 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/.tmp/info/bacf4a03ea0045b1ad2dd84ff193918b is 45, key is default/info:d/1700357408114/Put/seqid=0 2023-11-19T01:31:15,586 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:31:15,786 DEBUG [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 8af47d28058a302c1449b0cfff8bfc3d 2023-11-19T01:31:15,812 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/.tmp/info/c7ece878f671440ca2ea46332c384c14 2023-11-19T01:31:15,813 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/.tmp/info/bacf4a03ea0045b1ad2dd84ff193918b 2023-11-19T01:31:15,835 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/.tmp/info/bacf4a03ea0045b1ad2dd84ff193918b as hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/info/bacf4a03ea0045b1ad2dd84ff193918b 2023-11-19T01:31:15,841 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/info/bacf4a03ea0045b1ad2dd84ff193918b, entries=2, sequenceid=6, filesize=4.9 K 2023-11-19T01:31:15,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 8af47d28058a302c1449b0cfff8bfc3d in 460ms, sequenceid=6, compaction requested=false 2023-11-19T01:31:15,851 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34159/tmp/wal/data/hbase/namespace/8af47d28058a302c1449b0cfff8bfc3d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-11-19T01:31:15,851 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:15,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:31:15,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8af47d28058a302c1449b0cfff8bfc3d: 2023-11-19T01:31:15,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1700357407297.8af47d28058a302c1449b0cfff8bfc3d. 2023-11-19T01:31:15,857 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/.tmp/table/fdadd0b654b84904b79c62f5e5b1338e is 51, key is hbase:namespace/table:state/1700357408092/Put/seqid=0 2023-11-19T01:31:15,863 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/.tmp/table/fdadd0b654b84904b79c62f5e5b1338e 2023-11-19T01:31:15,869 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/.tmp/info/c7ece878f671440ca2ea46332c384c14 as hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/info/c7ece878f671440ca2ea46332c384c14 2023-11-19T01:31:15,875 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/info/c7ece878f671440ca2ea46332c384c14, entries=10, sequenceid=9, filesize=6.4 K 2023-11-19T01:31:15,876 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/.tmp/table/fdadd0b654b84904b79c62f5e5b1338e as hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/table/fdadd0b654b84904b79c62f5e5b1338e 2023-11-19T01:31:15,881 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/data/hbase/meta/1588230740/table/fdadd0b654b84904b79c62f5e5b1338e, entries=2, sequenceid=9, filesize=5.1 K 2023-11-19T01:31:15,882 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~1.26 KB/1290, heapSize ~2.61 KB/2672, currentSize=0 B/0 for 1588230740 in 500ms, sequenceid=9, compaction requested=false 2023-11-19T01:31:15,893 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34159/tmp/wal/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-11-19T01:31:15,894 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:15,894 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:31:15,894 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:31:15,894 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:31:15,894 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-19T01:31:15,986 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,33353,1700357404870; all regions closed. 2023-11-19T01:31:15,993 DEBUG [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-11-19T01:31:15,993 INFO [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C33353%2C1700357404870.meta:.meta(num 1700357407222) 2023-11-19T01:31:15,998 DEBUG [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /tmp/wal/oldWALs 2023-11-19T01:31:15,998 INFO [RS:0;jenkins-hbase6:33353 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C33353%2C1700357404870:(num 1700357406227) 2023-11-19T01:31:15,998 DEBUG [RS:0;jenkins-hbase6:33353 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:15,998 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:31:15,999 INFO [RS:0;jenkins-hbase6:33353 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-19T01:31:15,999 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:31:16,001 INFO [RS:0;jenkins-hbase6:33353 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:33353 2023-11-19T01:31:16,008 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/rs/jenkins-hbase6.apache.org,33353,1700357404870 2023-11-19T01:31:16,008 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2/rs 2023-11-19T01:31:16,011 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase6.apache.org,33353,1700357404870] 2023-11-19T01:31:16,012 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase6.apache.org,33353,1700357404870; numProcessing=1 2023-11-19T01:31:16,013 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /2/draining/jenkins-hbase6.apache.org,33353,1700357404870 already deleted, retry=false 2023-11-19T01:31:16,013 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase6.apache.org,33353,1700357404870 expired; onlineServers=0 2023-11-19T01:31:16,013 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,42885,1700357404683' ***** 2023-11-19T01:31:16,013 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-19T01:31:16,015 DEBUG [M:0;jenkins-hbase6:42885 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376bb786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:31:16,015 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:31:16,030 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/2/master 2023-11-19T01:31:16,030 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/2 2023-11-19T01:31:16,031 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Set watcher on znode that does not yet exist, /2/master 2023-11-19T01:31:16,111 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:16,111 INFO [RS:0;jenkins-hbase6:33353 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,33353,1700357404870; zookeeper connection closed. 2023-11-19T01:31:16,111 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33353-0x10123a6de220005, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:16,114 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b91e8ba {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b91e8ba 2023-11-19T01:31:16,114 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-19T01:31:16,114 INFO [M:0;jenkins-hbase6:42885 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@1fd325fa{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:31:16,115 INFO [M:0;jenkins-hbase6:42885 {}] server.AbstractConnector(383): Stopped ServerConnector@4597b7cc{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:31:16,115 INFO [M:0;jenkins-hbase6:42885 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:31:16,116 INFO [M:0;jenkins-hbase6:42885 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6eb2633e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:31:16,118 INFO [M:0;jenkins-hbase6:42885 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2c81789e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/hadoop.log.dir/,STOPPED} 2023-11-19T01:31:16,118 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,42885,1700357404683 2023-11-19T01:31:16,118 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,42885,1700357404683; all regions closed. 2023-11-19T01:31:16,118 DEBUG [M:0;jenkins-hbase6:42885 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:16,118 INFO [M:0;jenkins-hbase6:42885 {}] master.HMaster(1666): Stopping master jetty server 2023-11-19T01:31:16,119 INFO [M:0;jenkins-hbase6:42885 {}] server.AbstractConnector(383): Stopped ServerConnector@353b8c75{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:31:16,120 DEBUG [M:0;jenkins-hbase6:42885 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-19T01:31:16,120 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-19T01:31:16,120 INFO [M:0;jenkins-hbase6:42885 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-19T01:31:16,120 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357406024 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357406024,5,FailOnTimeoutGroup] 2023-11-19T01:31:16,120 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357406024 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357406024,5,FailOnTimeoutGroup] 2023-11-19T01:31:16,120 DEBUG [M:0;jenkins-hbase6:42885 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7fff1338 to 127.0.0.1:53145 2023-11-19T01:31:16,120 DEBUG [M:0;jenkins-hbase6:42885 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:16,120 DEBUG [M:0;jenkins-hbase6:42885 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-19T01:31:16,121 INFO [M:0;jenkins-hbase6:42885 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase6:0 had [] on shutdown 2023-11-19T01:31:16,121 DEBUG [M:0;jenkins-hbase6:42885 {}] master.HMaster(1689): Stopping service threads 2023-11-19T01:31:16,121 INFO [M:0;jenkins-hbase6:42885 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-19T01:31:16,121 ERROR [M:0;jenkins-hbase6:42885 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-19T01:31:16,122 INFO [M:0;jenkins-hbase6:42885 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-19T01:31:16,122 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-19T01:31:16,122 DEBUG [M:0;jenkins-hbase6:42885 {}] zookeeper.ZKUtil(347): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Unable to get data of znode /2/master because node does not exist (not an error) 2023-11-19T01:31:16,122 WARN [M:0;jenkins-hbase6:42885 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-19T01:31:16,122 INFO [M:0;jenkins-hbase6:42885 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-19T01:31:16,123 INFO [M:0;jenkins-hbase6:42885 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-19T01:31:16,125 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:31:16,143 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:16,143 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:16,143 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:31:16,143 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:16,143 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.76 KB heapSize=33.30 KB 2023-11-19T01:31:16,161 DEBUG [M:0;jenkins-hbase6:42885 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c609dd7554a147548adbdcc591b9da71 is 88, key is hbase:meta,,1/info:sn/1700357407046/Put/seqid=0 2023-11-19T01:31:16,567 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c609dd7554a147548adbdcc591b9da71 2023-11-19T01:31:16,591 DEBUG [M:0;jenkins-hbase6:42885 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46099b8f07a14c7696a59ae8f397444b is 241, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1700357407280/Put/seqid=0 2023-11-19T01:31:16,967 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-19T01:31:16,995 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25.17 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46099b8f07a14c7696a59ae8f397444b 2023-11-19T01:31:17,000 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 46099b8f07a14c7696a59ae8f397444b 2023-11-19T01:31:17,013 DEBUG [M:0;jenkins-hbase6:42885 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/18765008ce13410bbf06c8228ba936ec is 82, key is jenkins-hbase6.apache.org,33353,1700357404870/rs:state/1700357406066/Put/seqid=0 2023-11-19T01:31:17,418 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/18765008ce13410bbf06c8228ba936ec 2023-11-19T01:31:17,423 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c609dd7554a147548adbdcc591b9da71 as hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c609dd7554a147548adbdcc591b9da71 2023-11-19T01:31:17,428 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c609dd7554a147548adbdcc591b9da71, entries=8, sequenceid=76, filesize=5.5 K 2023-11-19T01:31:17,429 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/46099b8f07a14c7696a59ae8f397444b as hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/46099b8f07a14c7696a59ae8f397444b 2023-11-19T01:31:17,434 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 46099b8f07a14c7696a59ae8f397444b 2023-11-19T01:31:17,434 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/46099b8f07a14c7696a59ae8f397444b, entries=8, sequenceid=76, filesize=5.4 K 2023-11-19T01:31:17,435 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/18765008ce13410bbf06c8228ba936ec as hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/18765008ce13410bbf06c8228ba936ec 2023-11-19T01:31:17,439 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34159/user/jenkins/test-data/4ae57c62-b11b-933f-d1b4-01df880405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/18765008ce13410bbf06c8228ba936ec, entries=1, sequenceid=76, filesize=5.1 K 2023-11-19T01:31:17,440 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(3022): Finished flush of dataSize ~25.76 KB/26381, heapSize ~33.01 KB/33800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=76, compaction requested=false 2023-11-19T01:31:17,448 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:17,448 DEBUG [M:0;jenkins-hbase6:42885 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:31:17,452 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:31:17,452 INFO [M:0;jenkins-hbase6:42885 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-19T01:31:17,452 INFO [M:0;jenkins-hbase6:42885 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-19T01:31:17,454 INFO [M:0;jenkins-hbase6:42885 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:42885 2023-11-19T01:31:17,456 DEBUG [M:0;jenkins-hbase6:42885 {}] zookeeper.RecoverableZooKeeper(224): Node /2/rs/jenkins-hbase6.apache.org,42885,1700357404683 already deleted, retry=false 2023-11-19T01:31:17,558 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:17,558 INFO [M:0;jenkins-hbase6:42885 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,42885,1700357404683; zookeeper connection closed. 2023-11-19T01:31:17,558 DEBUG [Listener at localhost/34903-EventThread {}] zookeeper.ZKWatcher(604): master:42885-0x10123a6de220004, quorum=127.0.0.1:53145, baseZNode=/2 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:17,561 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:31:17,580 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:31:17,689 WARN [BP-2002050867-172.31.2.20-1700357403356 heartbeating to localhost/127.0.0.1:34159 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:31:17,689 WARN [BP-2002050867-172.31.2.20-1700357403356 heartbeating to localhost/127.0.0.1:34159 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2002050867-172.31.2.20-1700357403356 (Datanode Uuid dd141294-c327-4d14-9c0c-b205fa3ed59a) service to localhost/127.0.0.1:34159 2023-11-19T01:31:17,692 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4/dfs/data/data5/current/BP-2002050867-172.31.2.20-1700357403356 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:17,692 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4/dfs/data/data6/current/BP-2002050867-172.31.2.20-1700357403356 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:17,697 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:31:17,705 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:31:17,808 WARN [BP-2002050867-172.31.2.20-1700357403356 heartbeating to localhost/127.0.0.1:34159 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:31:17,808 WARN [BP-2002050867-172.31.2.20-1700357403356 heartbeating to localhost/127.0.0.1:34159 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2002050867-172.31.2.20-1700357403356 (Datanode Uuid 49555abc-4d33-4685-be34-f5c5aeea2cee) service to localhost/127.0.0.1:34159 2023-11-19T01:31:17,809 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4/dfs/data/data3/current/BP-2002050867-172.31.2.20-1700357403356 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:17,809 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4/dfs/data/data4/current/BP-2002050867-172.31.2.20-1700357403356 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:17,813 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:31:17,820 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:31:17,924 WARN [BP-2002050867-172.31.2.20-1700357403356 heartbeating to localhost/127.0.0.1:34159 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:31:17,924 WARN [BP-2002050867-172.31.2.20-1700357403356 heartbeating to localhost/127.0.0.1:34159 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2002050867-172.31.2.20-1700357403356 (Datanode Uuid b5960f6b-4eec-41f3-a486-e55d15795eda) service to localhost/127.0.0.1:34159 2023-11-19T01:31:17,925 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4/dfs/data/data1/current/BP-2002050867-172.31.2.20-1700357403356 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:17,925 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/764eb130-6325-6a06-5313-0476a78fcd87/cluster_4575015c-c1b5-90af-0f46-8884dfbfb7d4/dfs/data/data2/current/BP-2002050867-172.31.2.20-1700357403356 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:17,966 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:31:18,103 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-19T01:31:18,103 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-19T01:31:18,103 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:31:18,103 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x59761e0b to 127.0.0.1:53145 2023-11-19T01:31:18,104 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:18,104 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-19T01:31:18,104 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] util.JVMClusterUtil(257): Found active master hash=1865367665, stopped=false 2023-11-19T01:31:18,104 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:31:18,106 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-11-19T01:31:18,106 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-11-19T01:31:18,106 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:31:18,106 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-19T01:31:18,106 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:31:18,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:31:18,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:31:18,106 DEBUG [Listener at jenkins-hbase6.apache.org/41013 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:18,107 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,41675,1700357398894' ***** 2023-11-19T01:31:18,107 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-19T01:31:18,107 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:31:18,112 INFO [RS:0;jenkins-hbase6:41675 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@35ed2944{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:31:18,112 INFO [RS:0;jenkins-hbase6:41675 {}] server.AbstractConnector(383): Stopped ServerConnector@4fc7eff7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:31:18,113 INFO [RS:0;jenkins-hbase6:41675 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:31:18,114 INFO [RS:0;jenkins-hbase6:41675 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@328c6d9b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:31:18,115 INFO [RS:0;jenkins-hbase6:41675 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@f5e5cc6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir/,STOPPED} 2023-11-19T01:31:18,115 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-19T01:31:18,115 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for 3808ea9229abbf0011d0e0727029fc0a 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for 7234af1d9c333fce0f1fb04662aacac5 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for 92f2c176e8984c832db821343de7a411 2023-11-19T01:31:18,116 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3808ea9229abbf0011d0e0727029fc0a, disabling compactions & flushes 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for 8673ecec9222769a503bdb3d24ac32cc 2023-11-19T01:31:18,116 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for 3de69d369d7dacc90031c653c9918ad3 2023-11-19T01:31:18,116 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for dac22350d297ecf0792f58710c9ec235 2023-11-19T01:31:18,116 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. after waiting 0 ms 2023-11-19T01:31:18,116 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:31:18,116 DEBUG [RS:0;jenkins-hbase6:41675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-19T01:31:18,116 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-19T01:31:18,119 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-19T01:31:18,119 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1583): Online Regions={3808ea9229abbf0011d0e0727029fc0a=test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a., 7234af1d9c333fce0f1fb04662aacac5=backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5., 92f2c176e8984c832db821343de7a411=hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411., 1588230740=hbase:meta,,1.1588230740, 8673ecec9222769a503bdb3d24ac32cc=ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc., 3de69d369d7dacc90031c653c9918ad3=ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3., dac22350d297ecf0792f58710c9ec235=ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235., fb436e957cd5ee14b630b03f76f4c9d3=backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.} 2023-11-19T01:31:18,120 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:31:18,120 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:31:18,120 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3808ea9229abbf0011d0e0727029fc0a, 3de69d369d7dacc90031c653c9918ad3, 7234af1d9c333fce0f1fb04662aacac5, 8673ecec9222769a503bdb3d24ac32cc, 92f2c176e8984c832db821343de7a411, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:18,120 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:31:18,120 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:31:18,120 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:31:18,120 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.51 KB heapSize=24 KB 2023-11-19T01:31:18,128 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/default/test-1700357412826/3808ea9229abbf0011d0e0727029fc0a/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-11-19T01:31:18,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:18,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:31:18,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3808ea9229abbf0011d0e0727029fc0a: 2023-11-19T01:31:18,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1700357412826,,1700357413373.3808ea9229abbf0011d0e0727029fc0a. 2023-11-19T01:31:18,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7234af1d9c333fce0f1fb04662aacac5, disabling compactions & flushes 2023-11-19T01:31:18,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:31:18,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:31:18,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. after waiting 0 ms 2023-11-19T01:31:18,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:31:18,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7234af1d9c333fce0f1fb04662aacac5 2/2 column families, dataSize=985 B heapSize=1.97 KB 2023-11-19T01:31:18,136 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/meta/55f4eb4c11674e28bc27ed7f4d529ed0 is 169, key is trslm:hdfs://localhost:34159/backupUT\x00test-1700357412826/meta:log-roll-map/1700357446693/Put/seqid=0 2023-11-19T01:31:18,143 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/info/f8c51bd50606448f95640514ab8ea91a is 159, key is ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc./info:regioninfo/1700357416410/Put/seqid=0 2023-11-19T01:31:18,183 INFO [regionserver/jenkins-hbase6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:31:18,321 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3de69d369d7dacc90031c653c9918ad3, 7234af1d9c333fce0f1fb04662aacac5, 8673ecec9222769a503bdb3d24ac32cc, 92f2c176e8984c832db821343de7a411, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:18,521 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3de69d369d7dacc90031c653c9918ad3, 7234af1d9c333fce0f1fb04662aacac5, 8673ecec9222769a503bdb3d24ac32cc, 92f2c176e8984c832db821343de7a411, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:18,541 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/meta/55f4eb4c11674e28bc27ed7f4d529ed0 2023-11-19T01:31:18,549 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.12 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/info/f8c51bd50606448f95640514ab8ea91a 2023-11-19T01:31:18,556 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/109203b16e09452097ff6e407cd02590 is 310, key is session:backup_1700357437536/session:context/1700357447115/Put/seqid=0 2023-11-19T01:31:18,567 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/rep_barrier/a25940c2b656493591edc5694f8101ed is 90, key is table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad./rep_barrier:/1700357437408/DeleteFamily/seqid=0 2023-11-19T01:31:18,721 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3de69d369d7dacc90031c653c9918ad3, 7234af1d9c333fce0f1fb04662aacac5, 8673ecec9222769a503bdb3d24ac32cc, 92f2c176e8984c832db821343de7a411, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:18,905 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-19T01:31:18,905 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-19T01:31:18,921 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3de69d369d7dacc90031c653c9918ad3, 7234af1d9c333fce0f1fb04662aacac5, 8673ecec9222769a503bdb3d24ac32cc, 92f2c176e8984c832db821343de7a411, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:18,960 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=534 B at sequenceid=27 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/109203b16e09452097ff6e407cd02590 2023-11-19T01:31:18,968 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/meta/55f4eb4c11674e28bc27ed7f4d529ed0 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/meta/55f4eb4c11674e28bc27ed7f4d529ed0 2023-11-19T01:31:18,971 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=172 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/rep_barrier/a25940c2b656493591edc5694f8101ed 2023-11-19T01:31:18,974 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/meta/55f4eb4c11674e28bc27ed7f4d529ed0, entries=4, sequenceid=27, filesize=5.5 K 2023-11-19T01:31:18,975 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/.tmp/session/109203b16e09452097ff6e407cd02590 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/109203b16e09452097ff6e407cd02590 2023-11-19T01:31:18,980 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/session/109203b16e09452097ff6e407cd02590, entries=2, sequenceid=27, filesize=5.3 K 2023-11-19T01:31:18,981 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~985 B/985, heapSize ~1.94 KB/1984, currentSize=0 B/0 for 7234af1d9c333fce0f1fb04662aacac5 in 849ms, sequenceid=27, compaction requested=true 2023-11-19T01:31:18,998 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system/7234af1d9c333fce0f1fb04662aacac5/recovered.edits/30.seqid, newMaxSeqId=30, maxSeqId=1 2023-11-19T01:31:18,999 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,000 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:31:19,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7234af1d9c333fce0f1fb04662aacac5: 2023-11-19T01:31:19,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1700357420397.7234af1d9c333fce0f1fb04662aacac5. 2023-11-19T01:31:19,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 92f2c176e8984c832db821343de7a411, disabling compactions & flushes 2023-11-19T01:31:19,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:31:19,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:31:19,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. after waiting 0 ms 2023-11-19T01:31:19,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:31:19,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 92f2c176e8984c832db821343de7a411 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-19T01:31:19,007 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/table/a9c2734120f049758c8bd479af707caa is 84, key is table1,,1700357432935.4a7c265223769d355ddd8fda3a6de8ad./table:/1700357437408/DeleteFamily/seqid=0 2023-11-19T01:31:19,011 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.22 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/table/a9c2734120f049758c8bd479af707caa 2023-11-19T01:31:19,015 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/.tmp/info/5fa5f99a078246fb8c1a6cf73a1472ad is 45, key is default/info:d/1700357403112/Put/seqid=0 2023-11-19T01:31:19,017 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/info/f8c51bd50606448f95640514ab8ea91a as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/info/f8c51bd50606448f95640514ab8ea91a 2023-11-19T01:31:19,021 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/.tmp/info/5fa5f99a078246fb8c1a6cf73a1472ad 2023-11-19T01:31:19,023 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/info/f8c51bd50606448f95640514ab8ea91a, entries=74, sequenceid=63, filesize=14.4 K 2023-11-19T01:31:19,024 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/rep_barrier/a25940c2b656493591edc5694f8101ed as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/rep_barrier/a25940c2b656493591edc5694f8101ed 2023-11-19T01:31:19,026 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/.tmp/info/5fa5f99a078246fb8c1a6cf73a1472ad as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/info/5fa5f99a078246fb8c1a6cf73a1472ad 2023-11-19T01:31:19,029 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/rep_barrier/a25940c2b656493591edc5694f8101ed, entries=2, sequenceid=63, filesize=5.4 K 2023-11-19T01:31:19,030 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/.tmp/table/a9c2734120f049758c8bd479af707caa as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/table/a9c2734120f049758c8bd479af707caa 2023-11-19T01:31:19,032 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/info/5fa5f99a078246fb8c1a6cf73a1472ad, entries=7, sequenceid=11, filesize=5.1 K 2023-11-19T01:31:19,033 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 92f2c176e8984c832db821343de7a411 in 32ms, sequenceid=11, compaction requested=false 2023-11-19T01:31:19,033 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-19T01:31:19,041 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/table/a9c2734120f049758c8bd479af707caa, entries=17, sequenceid=63, filesize=6.0 K 2023-11-19T01:31:19,042 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.51 KB/13834, heapSize ~23.95 KB/24528, currentSize=0 B/0 for 1588230740 in 922ms, sequenceid=63, compaction requested=false 2023-11-19T01:31:19,042 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/namespace/92f2c176e8984c832db821343de7a411/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-19T01:31:19,042 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-19T01:31:19,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,044 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:31:19,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 92f2c176e8984c832db821343de7a411: 2023-11-19T01:31:19,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1700357401711.92f2c176e8984c832db821343de7a411. 2023-11-19T01:31:19,045 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8673ecec9222769a503bdb3d24ac32cc, disabling compactions & flushes 2023-11-19T01:31:19,046 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:31:19,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:31:19,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. after waiting 0 ms 2023-11-19T01:31:19,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:31:19,048 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8673ecec9222769a503bdb3d24ac32cc 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:31:19,058 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/hbase/meta/1588230740/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=1 2023-11-19T01:31:19,059 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,059 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:31:19,060 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:31:19,060 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:31:19,061 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-19T01:31:19,074 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/.tmp/f/25ae63a6b46b43b89e4804ad3e26074c is 37, key is row10/f:q1/1700357416744/Put/seqid=0 2023-11-19T01:31:19,121 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-11-19T01:31:19,122 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1583): Online Regions={8673ecec9222769a503bdb3d24ac32cc=ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc., 3de69d369d7dacc90031c653c9918ad3=ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3., dac22350d297ecf0792f58710c9ec235=ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235., fb436e957cd5ee14b630b03f76f4c9d3=backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3.} 2023-11-19T01:31:19,122 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 3de69d369d7dacc90031c653c9918ad3, 8673ecec9222769a503bdb3d24ac32cc, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:19,127 INFO [Socket Reader #1 for port 0 {}] ipc.Server$Connection(2001): Auth successful for appattempt_1700357450370_0001_000001 (auth:SIMPLE) 2023-11-19T01:31:19,322 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on 3de69d369d7dacc90031c653c9918ad3, 8673ecec9222769a503bdb3d24ac32cc, dac22350d297ecf0792f58710c9ec235, fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:19,480 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/.tmp/f/25ae63a6b46b43b89e4804ad3e26074c 2023-11-19T01:31:19,486 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/.tmp/f/25ae63a6b46b43b89e4804ad3e26074c as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/f/25ae63a6b46b43b89e4804ad3e26074c 2023-11-19T01:31:19,490 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/f/25ae63a6b46b43b89e4804ad3e26074c, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:31:19,491 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 8673ecec9222769a503bdb3d24ac32cc in 443ms, sequenceid=103, compaction requested=false 2023-11-19T01:31:19,503 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns2/test-17003574128261/8673ecec9222769a503bdb3d24ac32cc/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-19T01:31:19,504 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,505 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:31:19,505 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8673ecec9222769a503bdb3d24ac32cc: 2023-11-19T01:31:19,505 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17003574128261,,1700357415615.8673ecec9222769a503bdb3d24ac32cc. 2023-11-19T01:31:19,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3de69d369d7dacc90031c653c9918ad3, disabling compactions & flushes 2023-11-19T01:31:19,506 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:31:19,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:31:19,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. after waiting 0 ms 2023-11-19T01:31:19,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:31:19,509 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns4/test-17003574128263/3de69d369d7dacc90031c653c9918ad3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:31:19,510 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,511 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:31:19,511 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3de69d369d7dacc90031c653c9918ad3: 2023-11-19T01:31:19,511 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17003574128263,,1700357418949.3de69d369d7dacc90031c653c9918ad3. 2023-11-19T01:31:19,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing dac22350d297ecf0792f58710c9ec235, disabling compactions & flushes 2023-11-19T01:31:19,513 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:31:19,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:31:19,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. after waiting 0 ms 2023-11-19T01:31:19,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:31:19,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/ns3/test-17003574128262/dac22350d297ecf0792f58710c9ec235/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:31:19,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,517 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:31:19,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for dac22350d297ecf0792f58710c9ec235: 2023-11-19T01:31:19,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17003574128262,,1700357416823.dac22350d297ecf0792f58710c9ec235. 2023-11-19T01:31:19,518 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fb436e957cd5ee14b630b03f76f4c9d3, disabling compactions & flushes 2023-11-19T01:31:19,518 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:31:19,518 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:31:19,518 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. after waiting 0 ms 2023-11-19T01:31:19,518 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:31:19,521 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/data/backup/system_bulk/fb436e957cd5ee14b630b03f76f4c9d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:31:19,522 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:31:19,523 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:31:19,523 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fb436e957cd5ee14b630b03f76f4c9d3: 2023-11-19T01:31:19,523 DEBUG [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1609): Waiting on fb436e957cd5ee14b630b03f76f4c9d3 2023-11-19T01:31:19,523 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1700357422527.fb436e957cd5ee14b630b03f76f4c9d3. 2023-11-19T01:31:19,723 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,41675,1700357398894; all regions closed. 2023-11-19T01:31:19,731 DEBUG [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/oldWALs 2023-11-19T01:31:19,731 INFO [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C41675%2C1700357398894.meta:.meta(num 1700357441728) 2023-11-19T01:31:19,738 DEBUG [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/oldWALs 2023-11-19T01:31:19,738 INFO [RS:0;jenkins-hbase6:41675 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C41675%2C1700357398894:(num 1700357441711) 2023-11-19T01:31:19,738 DEBUG [RS:0;jenkins-hbase6:41675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:19,738 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:31:19,738 INFO [RS:0;jenkins-hbase6:41675 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase6:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-11-19T01:31:19,739 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:31:19,741 INFO [RS:0;jenkins-hbase6:41675 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:41675 2023-11-19T01:31:19,746 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase6.apache.org,41675,1700357398894 2023-11-19T01:31:19,746 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-19T01:31:19,748 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase6.apache.org,41675,1700357398894] 2023-11-19T01:31:19,748 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase6.apache.org,41675,1700357398894; numProcessing=1 2023-11-19T01:31:19,750 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase6.apache.org,41675,1700357398894 already deleted, retry=false 2023-11-19T01:31:19,750 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase6.apache.org,41675,1700357398894 expired; onlineServers=0 2023-11-19T01:31:19,750 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,34461,1700357395839' ***** 2023-11-19T01:31:19,750 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-19T01:31:19,751 DEBUG [M:0;jenkins-hbase6:34461 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a197e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:31:19,751 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:31:19,767 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-11-19T01:31:19,767 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:31:19,767 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:31:19,780 INFO [M:0;jenkins-hbase6:34461 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@165763a1{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:31:19,780 INFO [M:0;jenkins-hbase6:34461 {}] server.AbstractConnector(383): Stopped ServerConnector@6417d641{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:31:19,780 INFO [M:0;jenkins-hbase6:34461 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:31:19,781 INFO [M:0;jenkins-hbase6:34461 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3b083840{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:31:19,782 INFO [M:0;jenkins-hbase6:34461 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@41083923{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/hadoop.log.dir/,STOPPED} 2023-11-19T01:31:19,783 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,34461,1700357395839 2023-11-19T01:31:19,783 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,34461,1700357395839; all regions closed. 2023-11-19T01:31:19,783 DEBUG [M:0;jenkins-hbase6:34461 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:19,783 INFO [M:0;jenkins-hbase6:34461 {}] master.HMaster(1666): Stopping master jetty server 2023-11-19T01:31:19,784 INFO [M:0;jenkins-hbase6:34461 {}] server.AbstractConnector(383): Stopped ServerConnector@5df9048b{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:31:19,784 DEBUG [M:0;jenkins-hbase6:34461 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-19T01:31:19,784 INFO [M:0;jenkins-hbase6:34461 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-19T01:31:19,784 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-19T01:31:19,784 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357400663 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357400663,5,FailOnTimeoutGroup] 2023-11-19T01:31:19,784 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357400668 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357400668,5,FailOnTimeoutGroup] 2023-11-19T01:31:19,784 DEBUG [M:0;jenkins-hbase6:34461 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c69ecbb to 127.0.0.1:53145 2023-11-19T01:31:19,784 DEBUG [M:0;jenkins-hbase6:34461 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:19,784 DEBUG [M:0;jenkins-hbase6:34461 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-19T01:31:19,785 INFO [M:0;jenkins-hbase6:34461 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase6:0 had [] on shutdown 2023-11-19T01:31:19,785 DEBUG [M:0;jenkins-hbase6:34461 {}] master.HMaster(1689): Stopping service threads 2023-11-19T01:31:19,785 INFO [M:0;jenkins-hbase6:34461 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-19T01:31:19,785 ERROR [M:0;jenkins-hbase6:34461 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-2,5,PEWorkerGroup] 2023-11-19T01:31:19,785 INFO [M:0;jenkins-hbase6:34461 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-19T01:31:19,785 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-19T01:31:19,786 DEBUG [M:0;jenkins-hbase6:34461 {}] zookeeper.ZKUtil(347): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-11-19T01:31:19,786 WARN [M:0;jenkins-hbase6:34461 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-19T01:31:19,786 INFO [M:0;jenkins-hbase6:34461 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-19T01:31:19,786 INFO [M:0;jenkins-hbase6:34461 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-19T01:31:19,787 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:31:19,801 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:19,801 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:19,801 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:31:19,801 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:19,801 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=203.21 KB heapSize=244.17 KB 2023-11-19T01:31:19,813 DEBUG [M:0;jenkins-hbase6:34461 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e96946f8c62c4a6db6127c2c26a7a545 is 88, key is hbase:meta,,1/info:sn/1700357401153/Put/seqid=0 2023-11-19T01:31:19,848 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:19,848 INFO [RS:0;jenkins-hbase6:41675 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,41675,1700357398894; zookeeper connection closed. 2023-11-19T01:31:19,848 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): regionserver:41675-0x10123a6de220001, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:19,848 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2467aac5 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2467aac5 2023-11-19T01:31:19,848 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-19T01:31:20,234 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e96946f8c62c4a6db6127c2c26a7a545 2023-11-19T01:31:20,254 DEBUG [M:0;jenkins-hbase6:34461 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6e293d860804e4594812d1d6e8706ac is 988, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1700357423721/Put/seqid=0 2023-11-19T01:31:20,659 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202.62 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6e293d860804e4594812d1d6e8706ac 2023-11-19T01:31:20,664 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d6e293d860804e4594812d1d6e8706ac 2023-11-19T01:31:20,677 DEBUG [M:0;jenkins-hbase6:34461 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a0b70a050e640f7b825bed07cc2cd05 is 82, key is jenkins-hbase6.apache.org,41675,1700357398894/rs:state/1700357400814/Put/seqid=0 2023-11-19T01:31:20,681 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a0b70a050e640f7b825bed07cc2cd05 2023-11-19T01:31:20,687 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e96946f8c62c4a6db6127c2c26a7a545 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e96946f8c62c4a6db6127c2c26a7a545 2023-11-19T01:31:20,692 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e96946f8c62c4a6db6127c2c26a7a545, entries=8, sequenceid=480, filesize=5.5 K 2023-11-19T01:31:20,693 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6e293d860804e4594812d1d6e8706ac as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d6e293d860804e4594812d1d6e8706ac 2023-11-19T01:31:20,698 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d6e293d860804e4594812d1d6e8706ac 2023-11-19T01:31:20,698 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d6e293d860804e4594812d1d6e8706ac, entries=53, sequenceid=480, filesize=16.7 K 2023-11-19T01:31:20,699 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a0b70a050e640f7b825bed07cc2cd05 as hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6a0b70a050e640f7b825bed07cc2cd05 2023-11-19T01:31:20,702 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38223/user/jenkins/test-data/91ada3f3-b7fd-732a-07ef-e0ab035a08c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6a0b70a050e640f7b825bed07cc2cd05, entries=1, sequenceid=480, filesize=5.1 K 2023-11-19T01:31:20,703 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(3022): Finished flush of dataSize ~203.21 KB/208090, heapSize ~243.88 KB/249728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 902ms, sequenceid=480, compaction requested=false 2023-11-19T01:31:20,715 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:31:20,715 DEBUG [M:0;jenkins-hbase6:34461 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:31:20,719 INFO [M:0;jenkins-hbase6:34461 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-19T01:31:20,719 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:31:20,719 INFO [M:0;jenkins-hbase6:34461 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-19T01:31:20,721 INFO [M:0;jenkins-hbase6:34461 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:34461 2023-11-19T01:31:20,723 DEBUG [M:0;jenkins-hbase6:34461 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase6.apache.org,34461,1700357395839 already deleted, retry=false 2023-11-19T01:31:20,744 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:31:20,825 INFO [M:0;jenkins-hbase6:34461 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,34461,1700357395839; zookeeper connection closed. 2023-11-19T01:31:20,825 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:20,825 DEBUG [Listener at localhost/42919-EventThread {}] zookeeper.ZKWatcher(604): master:34461-0x10123a6de220000, quorum=127.0.0.1:53145, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-19T01:31:20,839 WARN [Listener at jenkins-hbase6.apache.org/41013 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:31:20,855 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:31:20,925 INFO [Finalizer {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:31:20,925 DEBUG [Finalizer {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x73fe804a to 127.0.0.1:53145 2023-11-19T01:31:20,925 DEBUG [Finalizer {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:31:21,003 WARN [BP-71530584-172.31.2.20-1700357392544 heartbeating to localhost/127.0.0.1:38223 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:31:21,003 WARN [BP-71530584-172.31.2.20-1700357392544 heartbeating to localhost/127.0.0.1:38223 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-71530584-172.31.2.20-1700357392544 (Datanode Uuid bee95ccc-1747-497c-9846-95b6ca0b696a) service to localhost/127.0.0.1:38223 2023-11-19T01:31:21,003 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/dfs/data/data1/current/BP-71530584-172.31.2.20-1700357392544 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:21,004 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/5df673cd-0176-4c71-54ed-f5c2b9617d06/cluster_147b8c80-7f1d-2f88-1fc3-e9031e13b9f7/dfs/data/data2/current/BP-71530584-172.31.2.20-1700357392544 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:31:21,014 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:31:21,130 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-11-19T01:31:21,155 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-19T01:31:21,155 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-19T01:31:21,167 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:31:21,285 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:31:21,398 ERROR [Thread[Thread-492,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:31:21,399 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:31:21,504 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-19T01:31:21,510 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-19T01:31:21,511 ERROR [Thread[Thread-502,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:31:21,519 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:31:21,626 ERROR [Thread[Thread-467,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:31:21,626 INFO [Listener at jenkins-hbase6.apache.org/41013 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped