2023-10-02 12:26:44,122 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-10-02 12:26:44,138 main DEBUG Took 0.013082 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:26:44,138 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:26:44,139 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:26:44,139 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:26:44,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,151 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:26:44,167 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,169 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,170 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,170 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,172 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,172 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,173 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,175 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,175 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,176 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,177 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,177 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:26:44,178 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,178 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:26:44,180 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:26:44,184 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:26:44,186 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:26:44,187 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:26:44,188 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:26:44,189 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:26:44,206 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:26:44,210 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:26:44,215 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:26:44,216 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:26:44,217 main DEBUG createAppenders(={Console}) 2023-10-02 12:26:44,218 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-10-02 12:26:44,218 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-10-02 12:26:44,219 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-10-02 12:26:44,219 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:26:44,220 main DEBUG OutputStream closed 2023-10-02 12:26:44,220 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:26:44,220 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:26:44,221 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-10-02 12:26:44,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:26:44,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:26:44,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:26:44,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:26:44,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:26:44,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:26:44,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:26:44,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:26:44,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:26:44,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:26:44,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:26:44,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:26:44,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:26:44,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:26:44,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:26:44,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:26:44,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:26:44,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:26:44,281 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:26:44,281 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-10-02 12:26:44,281 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:26:44,282 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-10-02T12:26:44,314 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupRepair timeout: 13 mins 2023-10-02 12:26:44,320 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:26:44,320 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:26:44,709 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291 2023-10-02T12:26:44,717 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:26:44,720 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:26:44,721 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:26:44,725 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06, deleteOnExit=true 2023-10-02T12:26:44,726 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:26:44,726 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/test.cache.data in system properties and HBase conf 2023-10-02T12:26:44,727 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:26:44,727 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:26:44,727 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:26:44,727 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:26:44,728 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:26:44,881 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:26:45,514 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:26:45,519 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:26:45,520 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:26:45,520 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:26:45,520 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:26:45,521 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:26:45,521 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:26:45,521 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:26:45,522 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:26:45,522 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:26:45,522 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:26:45,523 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:26:45,523 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:26:45,523 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:26:45,523 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:26:46,252 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:26:46,259 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:26:46,635 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:26:46,878 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@217ad086 via org.mortbay.log.Slf4jLog 2023-10-02T12:26:46,906 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:26:46,967 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:26:47,044 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir/Jetty_localhost_45241_hdfs____nxi94o/webapp 2023-10-02T12:26:47,353 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45241 2023-10-02T12:26:47,393 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:26:47,393 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:26:48,319 WARN [Listener at localhost/34623 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:26:48,509 WARN [Listener at localhost/34623 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:26:48,538 WARN [Listener at localhost/34623 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:26:48,547 INFO [Listener at localhost/34623 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:26:48,557 INFO [Listener at localhost/34623 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir/Jetty_localhost_36257_datanode____.erzzig/webapp 2023-10-02T12:26:48,718 INFO [Listener at localhost/34623 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36257 2023-10-02T12:26:49,159 WARN [Listener at localhost/42979 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:26:49,626 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:26:49,903 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc744ce5d8a6a3034: Processing first storage report for DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620 from datanode 872d4aba-ee6e-4db2-8b67-32a5a5f4eaf8 2023-10-02T12:26:49,905 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc744ce5d8a6a3034: from storage DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620 node DatanodeRegistration(127.0.0.1:40443, datanodeUuid=872d4aba-ee6e-4db2-8b67-32a5a5f4eaf8, infoPort=38407, infoSecurePort=0, ipcPort=42979, storageInfo=lv=-57;cid=testClusterID;nsid=904187423;c=1696249606345), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-10-02T12:26:49,905 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc744ce5d8a6a3034: Processing first storage report for DS-3e4a5204-998a-456f-92aa-cc1e445c2b12 from datanode 872d4aba-ee6e-4db2-8b67-32a5a5f4eaf8 2023-10-02T12:26:49,905 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc744ce5d8a6a3034: from storage DS-3e4a5204-998a-456f-92aa-cc1e445c2b12 node DatanodeRegistration(127.0.0.1:40443, datanodeUuid=872d4aba-ee6e-4db2-8b67-32a5a5f4eaf8, infoPort=38407, infoSecurePort=0, ipcPort=42979, storageInfo=lv=-57;cid=testClusterID;nsid=904187423;c=1696249606345), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:26:49,935 DEBUG [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291 2023-10-02T12:26:50,093 INFO [Listener at localhost/42979 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/zookeeper_0, clientPort=50557, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:26:50,121 INFO [Listener at localhost/42979 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50557 2023-10-02T12:26:50,133 INFO [Listener at localhost/42979 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:50,136 INFO [Listener at localhost/42979 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:51,058 INFO [Listener at localhost/42979 {}] util.FSUtils(462): Created version file at hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe with version=8 2023-10-02T12:26:51,058 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/hbase-staging 2023-10-02T12:26:51,077 DEBUG [Listener at localhost/42979 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:26:51,077 DEBUG [Listener at localhost/42979 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:26:51,078 DEBUG [Listener at localhost/42979 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:26:51,078 DEBUG [Listener at localhost/42979 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:26:51,374 DEBUG [Listener at localhost/42979 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:26:52,652 INFO [Listener at localhost/42979 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:26:52,721 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:26:52,721 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:26:52,722 INFO [Listener at localhost/42979 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:26:52,722 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:26:52,722 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:26:52,940 INFO [Listener at localhost/42979 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:26:53,135 INFO [Listener at localhost/42979 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:26:53,162 DEBUG [Listener at localhost/42979 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:26:53,171 INFO [Listener at localhost/42979 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:26:53,244 DEBUG [Listener at localhost/42979 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 30280 (auto-detected) 2023-10-02T12:26:53,247 DEBUG [Listener at localhost/42979 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:26:53,317 INFO [Listener at localhost/42979 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:36809 2023-10-02T12:26:53,338 INFO [Listener at localhost/42979 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:53,342 INFO [Listener at localhost/42979 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:53,398 INFO [Listener at localhost/42979 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:36809 connecting to ZooKeeper ensemble=127.0.0.1:50557 2023-10-02T12:26:53,494 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:368090x0, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:26:53,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:36809-0x10173245fa60000 connected 2023-10-02T12:26:53,673 DEBUG [Listener at localhost/42979 {}] zookeeper.ZKUtil(113): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:26:53,691 DEBUG [Listener at localhost/42979 {}] zookeeper.ZKUtil(113): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:26:53,702 DEBUG [Listener at localhost/42979 {}] zookeeper.ZKUtil(113): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:26:53,724 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36809 2023-10-02T12:26:53,725 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36809 2023-10-02T12:26:53,730 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36809 2023-10-02T12:26:53,734 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36809 2023-10-02T12:26:53,736 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36809 2023-10-02T12:26:53,859 INFO [Listener at localhost/42979 {}] log.Log(170): Logging initialized @10512ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:26:54,076 INFO [Listener at localhost/42979 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:26:54,077 INFO [Listener at localhost/42979 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:26:54,078 INFO [Listener at localhost/42979 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:26:54,081 INFO [Listener at localhost/42979 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:26:54,081 INFO [Listener at localhost/42979 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:26:54,081 INFO [Listener at localhost/42979 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:26:54,086 INFO [Listener at localhost/42979 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:26:54,281 INFO [Listener at localhost/42979 {}] http.HttpServer(1219): Jetty bound to port 37197 2023-10-02T12:26:54,284 INFO [Listener at localhost/42979 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:26:54,355 INFO [Listener at localhost/42979 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:26:54,360 INFO [Listener at localhost/42979 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7260be41{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:26:54,361 INFO [Listener at localhost/42979 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:26:54,361 INFO [Listener at localhost/42979 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2fa0fe4a{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:26:54,468 INFO [Listener at localhost/42979 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:26:54,488 INFO [Listener at localhost/42979 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:26:54,488 INFO [Listener at localhost/42979 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:26:54,491 INFO [Listener at localhost/42979 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:26:54,505 INFO [Listener at localhost/42979 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:26:56,525 INFO [Listener at localhost/42979 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@150be744{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:26:56,551 INFO [Listener at localhost/42979 {}] server.AbstractConnector(333): Started ServerConnector@15a156df{HTTP/1.1, (http/1.1)}{0.0.0.0:37197} 2023-10-02T12:26:56,552 INFO [Listener at localhost/42979 {}] server.Server(415): Started @13205ms 2023-10-02T12:26:56,557 INFO [Listener at localhost/42979 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe, hbase.cluster.distributed=false 2023-10-02T12:26:56,682 INFO [Listener at localhost/42979 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:26:56,682 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:26:56,683 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:26:56,683 INFO [Listener at localhost/42979 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:26:56,683 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:26:56,683 INFO [Listener at localhost/42979 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:26:56,713 INFO [Listener at localhost/42979 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:26:56,716 INFO [Listener at localhost/42979 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:26:56,734 INFO [Listener at localhost/42979 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:33719 2023-10-02T12:26:56,753 INFO [Listener at localhost/42979 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:26:56,781 DEBUG [Listener at localhost/42979 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:26:56,799 INFO [Listener at localhost/42979 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:56,803 INFO [Listener at localhost/42979 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:56,811 INFO [Listener at localhost/42979 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:33719 connecting to ZooKeeper ensemble=127.0.0.1:50557 2023-10-02T12:26:56,828 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:337190x0, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:26:56,831 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:33719-0x10173245fa60001 connected 2023-10-02T12:26:56,835 DEBUG [Listener at localhost/42979 {}] zookeeper.ZKUtil(113): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:26:56,842 DEBUG [Listener at localhost/42979 {}] zookeeper.ZKUtil(113): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:26:56,844 DEBUG [Listener at localhost/42979 {}] zookeeper.ZKUtil(113): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:26:56,847 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33719 2023-10-02T12:26:56,847 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33719 2023-10-02T12:26:56,848 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33719 2023-10-02T12:26:56,849 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33719 2023-10-02T12:26:56,850 DEBUG [Listener at localhost/42979 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33719 2023-10-02T12:26:56,854 INFO [Listener at localhost/42979 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:26:56,854 INFO [Listener at localhost/42979 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:26:56,854 INFO [Listener at localhost/42979 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:26:56,865 INFO [Listener at localhost/42979 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:26:56,865 INFO [Listener at localhost/42979 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:26:56,866 INFO [Listener at localhost/42979 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:26:56,866 INFO [Listener at localhost/42979 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:26:56,871 INFO [Listener at localhost/42979 {}] http.HttpServer(1219): Jetty bound to port 44495 2023-10-02T12:26:56,871 INFO [Listener at localhost/42979 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:26:56,970 INFO [Listener at localhost/42979 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:26:56,971 INFO [Listener at localhost/42979 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4e44870a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:26:56,971 INFO [Listener at localhost/42979 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:26:56,972 INFO [Listener at localhost/42979 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@227fa1d8{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:26:56,993 INFO [Listener at localhost/42979 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:26:56,995 INFO [Listener at localhost/42979 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:26:56,995 INFO [Listener at localhost/42979 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:26:56,995 INFO [Listener at localhost/42979 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:26:57,004 INFO [Listener at localhost/42979 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:26:57,014 INFO [Listener at localhost/42979 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3b0c1ba4{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:26:57,026 INFO [Listener at localhost/42979 {}] server.AbstractConnector(333): Started ServerConnector@33644794{HTTP/1.1, (http/1.1)}{0.0.0.0:44495} 2023-10-02T12:26:57,026 INFO [Listener at localhost/42979 {}] server.Server(415): Started @13680ms 2023-10-02T12:26:57,055 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:26:57,102 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@31401045{HTTP/1.1, (http/1.1)}{0.0.0.0:35497} 2023-10-02T12:26:57,102 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @13756ms 2023-10-02T12:26:57,103 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:26:57,119 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:36809 2023-10-02T12:26:57,133 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:26:57,134 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:26:57,135 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:26:57,177 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:26:57,177 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:26:57,178 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:26:57,181 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:26:57,184 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:26:57,187 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,36809,1696249611352 from backup master directory 2023-10-02T12:26:57,202 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:26:57,206 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:26:57,206 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:26:57,207 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:26:57,211 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:26:57,211 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:26:57,216 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:26:57,230 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:26:57,532 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/hbase.id with ID: 483d7672-91fb-4fc8-a00f-1e499cc085d0 2023-10-02T12:26:57,738 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:26:57,790 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:26:57,790 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:26:58,015 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:26:58,018 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:26:58,052 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:26:58,052 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:26:58,054 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:26:58,066 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:26:58,068 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:26:58,631 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store 2023-10-02T12:26:58,682 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:26:58,683 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:26:58,683 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:26:58,684 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:26:58,684 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:26:58,684 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:26:58,684 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:26:58,684 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:26:58,684 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:26:58,695 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/.initializing 2023-10-02T12:26:58,696 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/WALs/jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:26:58,716 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:26:58,743 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36809%2C1696249611352, suffix=, logDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/WALs/jenkins-hbase3.apache.org,36809,1696249611352, archiveDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/oldWALs, maxLogs=10 2023-10-02T12:26:58,799 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/WALs/jenkins-hbase3.apache.org,36809,1696249611352/jenkins-hbase3.apache.org%2C36809%2C1696249611352.1696249618753, exclude list is [], retry=0 2023-10-02T12:26:58,861 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:26:58,872 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:26:59,019 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/WALs/jenkins-hbase3.apache.org,36809,1696249611352/jenkins-hbase3.apache.org%2C36809%2C1696249611352.1696249618753 2023-10-02T12:26:59,026 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:26:59,027 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:26:59,028 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:26:59,038 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,040 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,174 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:26:59,252 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:26:59,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:26:59,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:26:59,279 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:26:59,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:26:59,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,302 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:26:59,302 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:26:59,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:26:59,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:26:59,319 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:26:59,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:26:59,327 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,345 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,399 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:26:59,406 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:26:59,428 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:26:59,430 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10506192480, jitterRate=-0.021534577012062073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:26:59,441 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:26:59,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:26:59,562 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74532022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:26:59,698 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:26:59,742 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:26:59,742 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:26:59,746 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:26:59,748 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 2 msec 2023-10-02T12:26:59,757 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 8 msec 2023-10-02T12:26:59,757 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:26:59,884 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:26:59,912 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:26:59,922 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:26:59,933 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:26:59,946 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:26:59,962 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:26:59,967 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:26:59,995 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:27:00,005 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:27:00,006 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:27:00,009 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:27:00,054 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:27:00,066 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:27:00,087 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:27:00,087 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,088 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:27:00,088 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,110 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,36809,1696249611352, sessionid=0x10173245fa60000, setting cluster-up flag (Was=false) 2023-10-02T12:27:00,141 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:27:00,158 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,158 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,222 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:27:00,225 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:27:00,242 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,242 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,290 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:27:00,292 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:27:00,318 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,319 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:00,360 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:27:00,367 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:27:00,479 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:33719 2023-10-02T12:27:00,504 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(997): ClusterId : 483d7672-91fb-4fc8-a00f-1e499cc085d0 2023-10-02T12:27:00,507 INFO [RS:0;jenkins-hbase3:33719 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:27:00,511 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:27:00,535 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:27:00,536 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:27:00,543 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:27:00,543 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:27:00,555 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:27:00,560 DEBUG [RS:0;jenkins-hbase3:33719 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bf2e01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:00,568 DEBUG [RS:0;jenkins-hbase3:33719 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d896a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:27:00,576 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:27:00,576 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:27:00,577 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:27:00,586 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,36809,1696249611352 with isa=jenkins-hbase3.apache.org/172.31.12.81:33719, startcode=1696249616681 2023-10-02T12:27:00,625 DEBUG [RS:0;jenkins-hbase3:33719 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:27:00,774 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46109, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:27:00,809 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:27:00,895 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:27:00,901 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-10-02T12:27:00,902 WARN [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-10-02T12:27:00,908 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:27:00,916 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:27:00,929 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,36809,1696249611352 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:27:00,946 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:27:00,946 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:27:00,947 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:27:00,947 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:27:00,947 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:27:00,947 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:00,952 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:27:00,953 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:00,990 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696249650990 2023-10-02T12:27:00,994 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:27:01,005 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:27:01,018 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,36809,1696249611352 with isa=jenkins-hbase3.apache.org/172.31.12.81:33719, startcode=1696249616681 2023-10-02T12:27:01,020 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:27:01,021 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:27:01,021 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:27:01,030 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-10-02T12:27:01,030 WARN [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 200 ms and then retrying. 2023-10-02T12:27:01,039 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:27:01,039 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:27:01,042 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:27:01,047 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:01,048 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:27:01,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:27:01,051 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:27:01,059 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,061 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:27:01,064 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:27:01,064 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:27:01,087 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70b9509a to 127.0.0.1:50557 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:27:01,138 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:27:01,138 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:01,148 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be75ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:01,148 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:27:01,170 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:27:01,171 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:27:01,174 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249621173,5,FailOnTimeoutGroup] 2023-10-02T12:27:01,174 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249621174,5,FailOnTimeoutGroup] 2023-10-02T12:27:01,174 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,175 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:27:01,177 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,177 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,235 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,36809,1696249611352 with isa=jenkins-hbase3.apache.org/172.31.12.81:33719, startcode=1696249616681 2023-10-02T12:27:01,243 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36809 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:01,264 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:01,265 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:34623 2023-10-02T12:27:01,265 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=37197 2023-10-02T12:27:01,279 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:27:01,284 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,33719,1696249616681] 2023-10-02T12:27:01,294 DEBUG [RS:0;jenkins-hbase3:33719 {}] zookeeper.ZKUtil(111): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:01,294 WARN [RS:0;jenkins-hbase3:33719 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:27:01,295 INFO [RS:0;jenkins-hbase3:33719 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:27:01,295 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:01,365 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:27:01,385 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:27:01,411 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:27:01,417 INFO [RS:0;jenkins-hbase3:33719 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:27:01,417 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,419 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:27:01,449 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,449 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,450 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,450 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,450 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,450 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,450 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:27:01,451 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,451 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,451 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,451 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,452 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:27:01,452 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:27:01,452 DEBUG [RS:0;jenkins-hbase3:33719 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:27:01,458 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,458 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,458 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,458 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,458 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33719,1696249616681-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:27:01,490 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:27:01,494 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33719,1696249616681-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:01,526 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,33719,1696249616681 started 2023-10-02T12:27:01,526 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,33719,1696249616681, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:33719, sessionid=0x10173245fa60001 2023-10-02T12:27:01,527 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:27:01,527 DEBUG [RS:0;jenkins-hbase3:33719 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:01,538 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33719,1696249616681' 2023-10-02T12:27:01,538 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:27:01,546 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:27:01,554 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:27:01,555 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:27:01,555 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33719,1696249616681' 2023-10-02T12:27:01,555 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:27:01,555 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:27:01,556 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:27:01,556 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:27:01,556 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:27:01,557 DEBUG [RS:0;jenkins-hbase3:33719 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:01,557 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,33719,1696249616681' 2023-10-02T12:27:01,557 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:27:01,558 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:27:01,566 DEBUG [RS:0;jenkins-hbase3:33719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:27:01,566 INFO [RS:0;jenkins-hbase3:33719 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:27:01,566 INFO [RS:0;jenkins-hbase3:33719 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:27:01,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:01,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:27:01,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:27:01,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:01,616 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:27:01,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:27:01,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:27:01,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:01,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:27:01,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:27:01,630 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:27:01,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:01,635 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:27:01,638 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740 2023-10-02T12:27:01,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740 2023-10-02T12:27:01,654 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:27:01,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:27:01,690 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:01,694 INFO [RS:0;jenkins-hbase3:33719 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:27:01,694 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9619519040, jitterRate=-0.10411247611045837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:27:01,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:27:01,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:27:01,698 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:27:01,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:27:01,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:27:01,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:27:01,700 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:27:01,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:27:01,713 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:27:01,713 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:27:01,715 INFO [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33719%2C1696249616681, suffix=, logDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681, archiveDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/oldWALs, maxLogs=32 2023-10-02T12:27:01,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:27:01,758 DEBUG [RS:0;jenkins-hbase3:33719 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249621718, exclude list is [], retry=0 2023-10-02T12:27:01,799 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:27:01,837 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:27:01,863 INFO [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249621718 2023-10-02T12:27:01,864 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:27:01,867 DEBUG [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:27:02,019 DEBUG [jenkins-hbase3:36809 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:27:02,047 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:02,084 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,33719,1696249616681, state=OPENING 2023-10-02T12:27:02,118 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:27:02,126 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:02,130 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:02,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:27:02,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:27:02,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:02,417 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:02,450 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:27:02,478 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40424, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:27:02,524 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:27:02,525 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:27:02,530 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:27:02,547 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta, suffix=.meta, logDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681, archiveDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/oldWALs, maxLogs=32 2023-10-02T12:27:02,586 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249622549.meta, exclude list is [], retry=0 2023-10-02T12:27:02,612 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:27:02,641 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249622549.meta 2023-10-02T12:27:02,646 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:27:02,646 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:02,662 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:02,664 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:27:02,706 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:27:02,707 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:27:02,720 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:27:02,720 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:02,720 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:27:02,720 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:27:02,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:27:02,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:27:02,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:02,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:27:02,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:27:02,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:27:02,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:02,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:27:02,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:27:02,764 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:27:02,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:02,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:27:02,773 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740 2023-10-02T12:27:02,779 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740 2023-10-02T12:27:02,782 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:27:02,785 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:27:02,794 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11879082880, jitterRate=0.10632580518722534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:27:02,796 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:27:02,818 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696249622403 2023-10-02T12:27:02,862 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:02,865 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,33719,1696249616681, state=OPEN 2023-10-02T12:27:02,870 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:27:02,878 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:27:02,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:27:02,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:27:02,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,33719,1696249616681 in 742 msec 2023-10-02T12:27:02,886 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:27:02,886 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:27:02,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:27:02,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:27:02,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1520 sec 2023-10-02T12:27:03,211 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:27:03,212 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:03,241 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:03,241 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:27:03,241 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:27:03,241 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:27:03,242 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:27:03,242 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:03,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 2.2900 sec 2023-10-02T12:27:03,244 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696249623244, completionTime=-1 2023-10-02T12:27:03,245 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:27:03,245 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:27:03,351 DEBUG [hconnection-0x7fb7937b-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:03,371 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40430, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:03,402 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:27:03,402 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696249683402 2023-10-02T12:27:03,402 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696249743402 2023-10-02T12:27:03,403 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 157 msec 2023-10-02T12:27:03,473 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36809,1696249611352-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:03,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36809,1696249611352-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:03,475 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36809,1696249611352-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:03,478 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:36809, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:03,494 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:27:03,521 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:27:03,529 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:27:03,531 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:27:03,565 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:27:03,571 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:03,574 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:03,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:04,041 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c1335df7709a0befb7a49aa3f6cd3add, NAME => 'hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:04,072 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:04,072 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing c1335df7709a0befb7a49aa3f6cd3add, disabling compactions & flushes 2023-10-02T12:27:04,072 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,072 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,073 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. after waiting 0 ms 2023-10-02T12:27:04,073 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,073 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,073 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for c1335df7709a0befb7a49aa3f6cd3add: 2023-10-02T12:27:04,079 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:04,101 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696249624081"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249624081"}]},"ts":"1696249624081"} 2023-10-02T12:27:04,154 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:04,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:04,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249624158"}]},"ts":"1696249624158"} 2023-10-02T12:27:04,174 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:27:04,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=c1335df7709a0befb7a49aa3f6cd3add, ASSIGN}] 2023-10-02T12:27:04,186 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=c1335df7709a0befb7a49aa3f6cd3add, ASSIGN 2023-10-02T12:27:04,189 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=c1335df7709a0befb7a49aa3f6cd3add, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:04,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=c1335df7709a0befb7a49aa3f6cd3add, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:04,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure c1335df7709a0befb7a49aa3f6cd3add, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:04,519 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,521 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => c1335df7709a0befb7a49aa3f6cd3add, NAME => 'hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:04,522 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:04,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:04,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,531 INFO [StoreOpener-c1335df7709a0befb7a49aa3f6cd3add-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,536 INFO [StoreOpener-c1335df7709a0befb7a49aa3f6cd3add-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1335df7709a0befb7a49aa3f6cd3add columnFamilyName info 2023-10-02T12:27:04,536 DEBUG [StoreOpener-c1335df7709a0befb7a49aa3f6cd3add-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:04,537 INFO [StoreOpener-c1335df7709a0befb7a49aa3f6cd3add-1 {}] regionserver.HStore(324): Store=c1335df7709a0befb7a49aa3f6cd3add/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:04,540 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,542 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,547 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:04,558 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:04,571 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened c1335df7709a0befb7a49aa3f6cd3add; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11587257600, jitterRate=0.07914745807647705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:27:04,572 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for c1335df7709a0befb7a49aa3f6cd3add: 2023-10-02T12:27:04,579 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add., pid=6, masterSystemTime=1696249624507 2023-10-02T12:27:04,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=c1335df7709a0befb7a49aa3f6cd3add, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:04,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,594 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:04,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:27:04,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure c1335df7709a0befb7a49aa3f6cd3add, server=jenkins-hbase3.apache.org,33719,1696249616681 in 252 msec 2023-10-02T12:27:04,620 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:04,620 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249624620"}]},"ts":"1696249624620"} 2023-10-02T12:27:04,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:27:04,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=c1335df7709a0befb7a49aa3f6cd3add, ASSIGN in 429 msec 2023-10-02T12:27:04,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:27:04,648 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:04,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.1150 sec 2023-10-02T12:27:04,682 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:27:04,687 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:04,688 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:04,690 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33719-0x10173245fa60001, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:27:04,799 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:27:04,885 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:04,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 128 msec 2023-10-02T12:27:04,938 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:27:04,983 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:05,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 69 msec 2023-10-02T12:27:05,036 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:27:05,073 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:27:05,074 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 7.862sec 2023-10-02T12:27:05,076 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:27:05,079 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:27:05,080 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:27:05,081 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:27:05,081 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:27:05,085 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36809,1696249611352-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:27:05,087 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36809,1696249611352-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:27:05,121 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:27:05,123 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:27:05,130 DEBUG [Listener at localhost/42979 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x22badfe7 to 127.0.0.1:50557 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:27:05,152 DEBUG [Listener at localhost/42979 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0b1518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:05,159 DEBUG [Listener at localhost/42979 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:27:05,159 DEBUG [Listener at localhost/42979 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:27:05,191 DEBUG [hconnection-0x1c07e9e3-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:05,212 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40432, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:05,228 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:27:05,228 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:27:05,229 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/test.cache.data in system properties and HBase conf 2023-10-02T12:27:05,229 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:27:05,229 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:27:05,229 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:27:05,229 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:27:05,229 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:27:05,230 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:27:05,230 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:27:05,237 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:27:05,237 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:27:05,237 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:27:05,237 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:27:05,238 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:27:05,238 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:27:05,238 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:27:05,238 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:27:05,238 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:27:05,239 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:27:05,239 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:27:05,239 INFO [Listener at localhost/42979 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:27:07,377 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:27:07,387 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:27:08,011 WARN [Listener at localhost/42979 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:27:08,337 WARN [Thread-250 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:27:08,900 INFO [Thread-250 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:27:08,911 INFO [Thread-250 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40343_jobhistory____.i9t6ps/webapp 2023-10-02T12:27:09,197 INFO [Thread-250 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:27:09,198 INFO [Thread-250 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:27:09,198 INFO [Thread-250 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:27:09,203 INFO [Thread-250 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:27:09,327 INFO [Thread-250 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:09,501 INFO [Thread-250 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:10,184 INFO [Thread-250 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:27:10,319 INFO [Thread-250 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40343 2023-10-02T12:27:11,316 WARN [Listener at jenkins-hbase3.apache.org/45035 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:27:11,450 WARN [Listener at jenkins-hbase3.apache.org/45035 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:27:11,450 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:27:11,467 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41581_cluster____.r585vy/webapp 2023-10-02T12:27:11,612 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:27:11,612 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:27:11,613 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:27:11,618 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:27:11,632 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:11,970 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:12,161 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:12,191 INFO [Listener at jenkins-hbase3.apache.org/45035 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41581 2023-10-02T12:27:12,930 WARN [Listener at jenkins-hbase3.apache.org/37313 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:27:12,968 WARN [Listener at jenkins-hbase3.apache.org/37313 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:27:12,968 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:27:12,979 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41787_node____vdqzg8/webapp 2023-10-02T12:27:13,049 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:27:13,049 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:27:13,049 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:27:13,050 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:27:13,062 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:13,093 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:13,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:27:13,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:13,225 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:13,272 INFO [Listener at jenkins-hbase3.apache.org/37313 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41787 2023-10-02T12:27:13,878 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:27:13,893 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:27:13,894 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:27:13,907 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44175_node____.4zrq8i/webapp 2023-10-02T12:27:13,993 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:27:13,994 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:27:13,994 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:27:13,994 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:27:14,005 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:14,048 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:14,161 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:27:14,169 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44175 2023-10-02T12:27:14,253 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:27:14,253 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:34623/backupUT 2023-10-02T12:27:14,273 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:14,287 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58066, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:14,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:27:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:27:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:27:14,381 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:14,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 79 msec 2023-10-02T12:27:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:27:14,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:27:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:27:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:27:14,499 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:14,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 52 msec 2023-10-02T12:27:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:27:14,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:27:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:27:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:27:14,622 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:14,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 39 msec 2023-10-02T12:27:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:27:14,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:27:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:27:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:27:14,769 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:14,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 56 msec 2023-10-02T12:27:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:27:14,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696249634253', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:27:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696249634253 2023-10-02T12:27:14,893 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696249634253 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:14,894 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:14,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696249634253" procId is: 13 2023-10-02T12:27:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:27:14,908 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696249634253 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:27:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:27:15,449 INFO [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a863321cd7a76d54b557bc7d0d9b8156, NAME => 'test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696249634253', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:15,508 DEBUG [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:15,508 DEBUG [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1668): Closing a863321cd7a76d54b557bc7d0d9b8156, disabling compactions & flushes 2023-10-02T12:27:15,508 INFO [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,508 DEBUG [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,508 DEBUG [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. after waiting 0 ms 2023-10-02T12:27:15,508 DEBUG [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,508 INFO [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1909): Closed test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,508 DEBUG [RegionOpenAndInit-test-1696249634253-pool-0 {}] regionserver.HRegion(1622): Region close journal for a863321cd7a76d54b557bc7d0d9b8156: 2023-10-02T12:27:15,511 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696249634253 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:15,511 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696249635511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249635511"}]},"ts":"1696249635511"} 2023-10-02T12:27:15,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:27:15,524 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696249634253 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:15,524 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249634253","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249635524"}]},"ts":"1696249635524"} 2023-10-02T12:27:15,527 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249634253, state=ENABLING in hbase:meta 2023-10-02T12:27:15,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249634253, region=a863321cd7a76d54b557bc7d0d9b8156, ASSIGN}] 2023-10-02T12:27:15,538 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249634253, region=a863321cd7a76d54b557bc7d0d9b8156, ASSIGN 2023-10-02T12:27:15,540 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696249634253, region=a863321cd7a76d54b557bc7d0d9b8156, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:15,692 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=a863321cd7a76d54b557bc7d0d9b8156, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:15,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure a863321cd7a76d54b557bc7d0d9b8156, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:15,904 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,905 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => a863321cd7a76d54b557bc7d0d9b8156, NAME => 'test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:15,906 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:15,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696249634253 a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:15,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,911 INFO [StoreOpener-a863321cd7a76d54b557bc7d0d9b8156-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,914 INFO [StoreOpener-a863321cd7a76d54b557bc7d0d9b8156-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a863321cd7a76d54b557bc7d0d9b8156 columnFamilyName f 2023-10-02T12:27:15,914 DEBUG [StoreOpener-a863321cd7a76d54b557bc7d0d9b8156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:15,922 INFO [StoreOpener-a863321cd7a76d54b557bc7d0d9b8156-1 {}] regionserver.HStore(324): Store=a863321cd7a76d54b557bc7d0d9b8156/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:15,924 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,933 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:15,952 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:15,954 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened a863321cd7a76d54b557bc7d0d9b8156; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10622207040, jitterRate=-0.010729879140853882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:27:15,955 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for a863321cd7a76d54b557bc7d0d9b8156: 2023-10-02T12:27:15,957 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156., pid=15, masterSystemTime=1696249635860 2023-10-02T12:27:15,962 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,963 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:15,975 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=a863321cd7a76d54b557bc7d0d9b8156, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:15,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:27:15,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure a863321cd7a76d54b557bc7d0d9b8156, server=jenkins-hbase3.apache.org,33719,1696249616681 in 287 msec 2023-10-02T12:27:16,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:27:16,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696249634253, region=a863321cd7a76d54b557bc7d0d9b8156, ASSIGN in 464 msec 2023-10-02T12:27:16,003 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696249634253 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:16,003 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249634253","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249636003"}]},"ts":"1696249636003"} 2023-10-02T12:27:16,008 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249634253, state=ENABLED in hbase:meta 2023-10-02T12:27:16,031 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696249634253 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:27:16,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696249634253 in 1.1600 sec 2023-10-02T12:27:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:27:17,036 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696249634253, procId: 13 completed 2023-10-02T12:27:17,039 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x25ea649d to 127.0.0.1:50557 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:27:17,063 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319b580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:17,082 DEBUG [hconnection-0x153a4df7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:17,119 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42270, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:17,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] regionserver.HRegion(8210): writing data to region test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:27:17,416 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962496342531', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:27:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962496342531 2023-10-02T12:27:17,424 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962496342531 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:17,424 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:17,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962496342531" procId is: 16 2023-10-02T12:27:17,426 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962496342531 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:27:17,482 INFO [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2694c02ee78bcedb7abe52d8a1144cf9, NAME => 'ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962496342531', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:27:17,531 DEBUG [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:17,531 DEBUG [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1668): Closing 2694c02ee78bcedb7abe52d8a1144cf9, disabling compactions & flushes 2023-10-02T12:27:17,531 INFO [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,531 DEBUG [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,531 DEBUG [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. after waiting 0 ms 2023-10-02T12:27:17,531 DEBUG [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,531 INFO [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,531 DEBUG [RegionOpenAndInit-ns2:test-16962496342531-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2694c02ee78bcedb7abe52d8a1144cf9: 2023-10-02T12:27:17,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962496342531 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:17,535 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249637534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249637534"}]},"ts":"1696249637534"} 2023-10-02T12:27:17,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:17,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962496342531 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:17,544 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962496342531","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249637544"}]},"ts":"1696249637544"} 2023-10-02T12:27:17,550 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962496342531, state=ENABLING in hbase:meta 2023-10-02T12:27:17,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962496342531, region=2694c02ee78bcedb7abe52d8a1144cf9, ASSIGN}] 2023-10-02T12:27:17,559 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962496342531, region=2694c02ee78bcedb7abe52d8a1144cf9, ASSIGN 2023-10-02T12:27:17,561 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962496342531, region=2694c02ee78bcedb7abe52d8a1144cf9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:17,719 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=2694c02ee78bcedb7abe52d8a1144cf9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:17,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 2694c02ee78bcedb7abe52d8a1144cf9, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:27:17,883 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,883 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 2694c02ee78bcedb7abe52d8a1144cf9, NAME => 'ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:17,884 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:17,884 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962496342531 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,884 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:17,884 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,884 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,887 INFO [StoreOpener-2694c02ee78bcedb7abe52d8a1144cf9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,895 INFO [StoreOpener-2694c02ee78bcedb7abe52d8a1144cf9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2694c02ee78bcedb7abe52d8a1144cf9 columnFamilyName f 2023-10-02T12:27:17,895 DEBUG [StoreOpener-2694c02ee78bcedb7abe52d8a1144cf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:17,897 INFO [StoreOpener-2694c02ee78bcedb7abe52d8a1144cf9-1 {}] regionserver.HStore(324): Store=2694c02ee78bcedb7abe52d8a1144cf9/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:17,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,899 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,903 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:17,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:17,915 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 2694c02ee78bcedb7abe52d8a1144cf9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9677652640, jitterRate=-0.09869836270809174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:27:17,916 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 2694c02ee78bcedb7abe52d8a1144cf9: 2023-10-02T12:27:17,917 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9., pid=18, masterSystemTime=1696249637877 2023-10-02T12:27:17,921 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,921 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:17,922 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=2694c02ee78bcedb7abe52d8a1144cf9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:17,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:27:17,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 2694c02ee78bcedb7abe52d8a1144cf9, server=jenkins-hbase3.apache.org,33719,1696249616681 in 205 msec 2023-10-02T12:27:17,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:27:17,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962496342531, region=2694c02ee78bcedb7abe52d8a1144cf9, ASSIGN in 375 msec 2023-10-02T12:27:17,936 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962496342531 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:17,936 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962496342531","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249637936"}]},"ts":"1696249637936"} 2023-10-02T12:27:17,939 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962496342531, state=ENABLED in hbase:meta 2023-10-02T12:27:17,945 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962496342531 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:17,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962496342531 in 529 msec 2023-10-02T12:27:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:27:18,036 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962496342531, procId: 16 completed 2023-10-02T12:27:18,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33719 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:27:18,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962496342532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:27:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962496342532 2023-10-02T12:27:18,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962496342532 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:18,210 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:18,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962496342532" procId is: 19 2023-10-02T12:27:18,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962496342532 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:27:18,245 INFO [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 13109ef6404ab06943505340044ec89e, NAME => 'ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962496342532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:18,272 DEBUG [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:18,272 DEBUG [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1668): Closing 13109ef6404ab06943505340044ec89e, disabling compactions & flushes 2023-10-02T12:27:18,272 INFO [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,272 DEBUG [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,272 DEBUG [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. after waiting 0 ms 2023-10-02T12:27:18,272 DEBUG [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,272 INFO [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,272 DEBUG [RegionOpenAndInit-ns3:test-16962496342532-pool-0 {}] regionserver.HRegion(1622): Region close journal for 13109ef6404ab06943505340044ec89e: 2023-10-02T12:27:18,274 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962496342532 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:18,275 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249638274"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249638274"}]},"ts":"1696249638274"} 2023-10-02T12:27:18,278 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:18,279 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962496342532 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:18,280 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962496342532","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249638280"}]},"ts":"1696249638280"} 2023-10-02T12:27:18,284 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962496342532, state=ENABLING in hbase:meta 2023-10-02T12:27:18,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962496342532, region=13109ef6404ab06943505340044ec89e, ASSIGN}] 2023-10-02T12:27:18,293 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962496342532, region=13109ef6404ab06943505340044ec89e, ASSIGN 2023-10-02T12:27:18,294 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962496342532, region=13109ef6404ab06943505340044ec89e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:27:18,446 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=13109ef6404ab06943505340044ec89e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:18,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 13109ef6404ab06943505340044ec89e, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:27:18,634 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 13109ef6404ab06943505340044ec89e, NAME => 'ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:18,634 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:18,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962496342532 13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:18,635 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,635 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,637 INFO [StoreOpener-13109ef6404ab06943505340044ec89e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,639 INFO [StoreOpener-13109ef6404ab06943505340044ec89e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13109ef6404ab06943505340044ec89e columnFamilyName f 2023-10-02T12:27:18,639 DEBUG [StoreOpener-13109ef6404ab06943505340044ec89e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:18,640 INFO [StoreOpener-13109ef6404ab06943505340044ec89e-1 {}] regionserver.HStore(324): Store=13109ef6404ab06943505340044ec89e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:18,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns3/test-16962496342532/13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns3/test-16962496342532/13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,648 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 13109ef6404ab06943505340044ec89e 2023-10-02T12:27:18,653 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns3/test-16962496342532/13109ef6404ab06943505340044ec89e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:18,655 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 13109ef6404ab06943505340044ec89e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9773374880, jitterRate=-0.08978353440761566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:27:18,655 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 13109ef6404ab06943505340044ec89e: 2023-10-02T12:27:18,657 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e., pid=21, masterSystemTime=1696249638628 2023-10-02T12:27:18,660 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,660 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:18,661 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=13109ef6404ab06943505340044ec89e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:18,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:27:18,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 13109ef6404ab06943505340044ec89e, server=jenkins-hbase3.apache.org,33719,1696249616681 in 206 msec 2023-10-02T12:27:18,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:27:18,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962496342532, region=13109ef6404ab06943505340044ec89e, ASSIGN in 377 msec 2023-10-02T12:27:18,671 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962496342532 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:18,671 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962496342532","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249638671"}]},"ts":"1696249638671"} 2023-10-02T12:27:18,674 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962496342532, state=ENABLED in hbase:meta 2023-10-02T12:27:18,677 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962496342532 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:18,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962496342532 in 475 msec 2023-10-02T12:27:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:27:18,821 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962496342532, procId: 19 completed 2023-10-02T12:27:18,822 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962496342532 get assigned. Timeout = 60000ms 2023-10-02T12:27:18,825 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:27:18,831 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962496342532 assigned to meta. Checking AM states. 2023-10-02T12:27:18,832 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:27:18,832 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962496342532 assigned. 2023-10-02T12:27:18,835 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962496342533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:27:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962496342533 2023-10-02T12:27:18,840 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962496342533 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:18,840 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:18,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962496342533" procId is: 22 2023-10-02T12:27:18,842 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962496342533 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:27:18,860 INFO [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 55a233dd7aad88b70405150c81a513f0, NAME => 'ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962496342533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:18,891 DEBUG [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:18,891 DEBUG [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1668): Closing 55a233dd7aad88b70405150c81a513f0, disabling compactions & flushes 2023-10-02T12:27:18,891 INFO [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:18,891 DEBUG [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:18,891 DEBUG [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. after waiting 0 ms 2023-10-02T12:27:18,891 DEBUG [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:18,891 INFO [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:18,891 DEBUG [RegionOpenAndInit-ns4:test-16962496342533-pool-0 {}] regionserver.HRegion(1622): Region close journal for 55a233dd7aad88b70405150c81a513f0: 2023-10-02T12:27:18,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962496342533 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:18,893 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249638893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249638893"}]},"ts":"1696249638893"} 2023-10-02T12:27:18,899 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:18,901 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962496342533 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:18,901 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962496342533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249638901"}]},"ts":"1696249638901"} 2023-10-02T12:27:18,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962496342533, state=ENABLING in hbase:meta 2023-10-02T12:27:18,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962496342533, region=55a233dd7aad88b70405150c81a513f0, ASSIGN}] 2023-10-02T12:27:18,914 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962496342533, region=55a233dd7aad88b70405150c81a513f0, ASSIGN 2023-10-02T12:27:18,915 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962496342533, region=55a233dd7aad88b70405150c81a513f0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:27:19,067 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=55a233dd7aad88b70405150c81a513f0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:19,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 55a233dd7aad88b70405150c81a513f0, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:27:19,230 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:19,230 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 55a233dd7aad88b70405150c81a513f0, NAME => 'ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:19,231 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:19,231 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962496342533 55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,231 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:19,231 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,232 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,234 INFO [StoreOpener-55a233dd7aad88b70405150c81a513f0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,237 INFO [StoreOpener-55a233dd7aad88b70405150c81a513f0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55a233dd7aad88b70405150c81a513f0 columnFamilyName f 2023-10-02T12:27:19,237 DEBUG [StoreOpener-55a233dd7aad88b70405150c81a513f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:19,238 INFO [StoreOpener-55a233dd7aad88b70405150c81a513f0-1 {}] regionserver.HStore(324): Store=55a233dd7aad88b70405150c81a513f0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:19,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns4/test-16962496342533/55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,244 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns4/test-16962496342533/55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,249 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:19,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns4/test-16962496342533/55a233dd7aad88b70405150c81a513f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:19,255 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 55a233dd7aad88b70405150c81a513f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11694122560, jitterRate=0.08910003304481506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:27:19,256 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 55a233dd7aad88b70405150c81a513f0: 2023-10-02T12:27:19,259 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0., pid=24, masterSystemTime=1696249639224 2023-10-02T12:27:19,264 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:19,264 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:19,265 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=55a233dd7aad88b70405150c81a513f0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:19,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:27:19,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 55a233dd7aad88b70405150c81a513f0, server=jenkins-hbase3.apache.org,33719,1696249616681 in 196 msec 2023-10-02T12:27:19,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:27:19,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962496342533, region=55a233dd7aad88b70405150c81a513f0, ASSIGN in 360 msec 2023-10-02T12:27:19,275 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962496342533 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:19,276 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962496342533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249639275"}]},"ts":"1696249639275"} 2023-10-02T12:27:19,278 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962496342533, state=ENABLED in hbase:meta 2023-10-02T12:27:19,282 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962496342533 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:19,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962496342533 in 448 msec 2023-10-02T12:27:19,389 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962496342532' 2023-10-02T12:27:19,390 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962496342531' 2023-10-02T12:27:19,390 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696249634253' 2023-10-02T12:27:19,391 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962496342533' 2023-10-02T12:27:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:27:19,450 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962496342533, procId: 22 completed 2023-10-02T12:27:19,450 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962496342533 get assigned. Timeout = 60000ms 2023-10-02T12:27:19,450 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:27:19,455 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962496342533 assigned to meta. Checking AM states. 2023-10-02T12:27:19,455 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:27:19,455 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962496342533 assigned. 2023-10-02T12:27:19,456 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x25ea649d to 127.0.0.1:50557 2023-10-02T12:27:19,456 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:19,505 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.ResourceChecker(147): before: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=763, ProcessCount=181, AvailableMemoryMB=1507 2023-10-02T12:27:19,505 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-10-02T12:27:19,511 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupRepair(57): Running stage 0 2023-10-02T12:27:19,532 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:27:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:27:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:27:19,552 DEBUG [Listener at localhost/42979-EventThread {}] zookeeper.ZKWatcher(604): master:36809-0x10173245fa60000, quorum=127.0.0.1:50557, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:27:19,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 24 msec 2023-10-02T12:27:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:27:19,666 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:27:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:27:19,671 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:19,671 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:19,672 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:27:19,673 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:27:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:27:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:27:20,110 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:20,125 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:20,125 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing bca8be4589edc7e4cee153d1c0f19ab5, disabling compactions & flushes 2023-10-02T12:27:20,125 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,125 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,126 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. after waiting 0 ms 2023-10-02T12:27:20,126 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,126 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,126 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:20,128 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:20,129 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249640128"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249640128"}]},"ts":"1696249640128"} 2023-10-02T12:27:20,131 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:20,132 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:20,133 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249640132"}]},"ts":"1696249640132"} 2023-10-02T12:27:20,135 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:27:20,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN}] 2023-10-02T12:27:20,143 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN 2023-10-02T12:27:20,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:27:20,299 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:20,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:20,460 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,460 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:20,461 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:20,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:20,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,464 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,467 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName meta 2023-10-02T12:27:20,468 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:20,469 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:20,469 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,471 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName session 2023-10-02T12:27:20,471 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:20,472 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:20,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:27:20,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:20,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:20,481 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened bca8be4589edc7e4cee153d1c0f19ab5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11042921120, jitterRate=0.028452172875404358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:27:20,481 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:20,483 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., pid=28, masterSystemTime=1696249640455 2023-10-02T12:27:20,485 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,485 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:20,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:20,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:27:20,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 186 msec 2023-10-02T12:27:20,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:27:20,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN in 352 msec 2023-10-02T12:27:20,496 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:20,496 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249640496"}]},"ts":"1696249640496"} 2023-10-02T12:27:20,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:27:20,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:20,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 836 msec 2023-10-02T12:27:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:27:20,811 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:27:20,832 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:27:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:27:20,836 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:27:20,836 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:20,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:27:20,837 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:27:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:27:20,856 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 60ae3f470b3b69f99aa07d4a8996765d, NAME => 'backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe 2023-10-02T12:27:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:27:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:27:21,282 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:21,282 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 60ae3f470b3b69f99aa07d4a8996765d, disabling compactions & flushes 2023-10-02T12:27:21,282 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,282 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,282 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. after waiting 0 ms 2023-10-02T12:27:21,282 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,282 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,282 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 60ae3f470b3b69f99aa07d4a8996765d: 2023-10-02T12:27:21,284 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:27:21,285 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696249641284"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249641284"}]},"ts":"1696249641284"} 2023-10-02T12:27:21,287 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:21,288 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:27:21,289 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249641288"}]},"ts":"1696249641288"} 2023-10-02T12:27:21,290 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:27:21,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=60ae3f470b3b69f99aa07d4a8996765d, ASSIGN}] 2023-10-02T12:27:21,297 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=60ae3f470b3b69f99aa07d4a8996765d, ASSIGN 2023-10-02T12:27:21,299 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=60ae3f470b3b69f99aa07d4a8996765d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,33719,1696249616681; forceNewPlan=false, retain=false 2023-10-02T12:27:21,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=60ae3f470b3b69f99aa07d4a8996765d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:27:21,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 60ae3f470b3b69f99aa07d4a8996765d, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:21,565 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:27:21,614 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,614 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 60ae3f470b3b69f99aa07d4a8996765d, NAME => 'backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:21,614 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:21,614 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:21,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,619 INFO [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,625 INFO [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60ae3f470b3b69f99aa07d4a8996765d columnFamilyName meta 2023-10-02T12:27:21,625 DEBUG [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:21,626 INFO [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] regionserver.HStore(324): Store=60ae3f470b3b69f99aa07d4a8996765d/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:21,626 INFO [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,628 INFO [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60ae3f470b3b69f99aa07d4a8996765d columnFamilyName session 2023-10-02T12:27:21,628 DEBUG [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:21,629 INFO [StoreOpener-60ae3f470b3b69f99aa07d4a8996765d-1 {}] regionserver.HStore(324): Store=60ae3f470b3b69f99aa07d4a8996765d/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:21,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system_bulk/60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,631 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system_bulk/60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:27:21,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:21,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system_bulk/60ae3f470b3b69f99aa07d4a8996765d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:27:21,643 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 60ae3f470b3b69f99aa07d4a8996765d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11407796640, jitterRate=0.06243385374546051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:27:21,644 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 60ae3f470b3b69f99aa07d4a8996765d: 2023-10-02T12:27:21,646 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d., pid=31, masterSystemTime=1696249641606 2023-10-02T12:27:21,648 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,648 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:21,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=60ae3f470b3b69f99aa07d4a8996765d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:21,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:27:21,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 60ae3f470b3b69f99aa07d4a8996765d, server=jenkins-hbase3.apache.org,33719,1696249616681 in 198 msec 2023-10-02T12:27:21,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:27:21,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=60ae3f470b3b69f99aa07d4a8996765d, ASSIGN in 359 msec 2023-10-02T12:27:21,659 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:27:21,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249641659"}]},"ts":"1696249641659"} 2023-10-02T12:27:21,661 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:27:21,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:27:21,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 833 msec 2023-10-02T12:27:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:27:21,954 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:27:22,045 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c57b53a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:22,049 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:22,052 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45328, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:22,060 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:22,072 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:22,232 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:27:22,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249642294 (current time:1696249642294). 2023-10-02T12:27:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:27:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d4d7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:22,315 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:27:22,343 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:27:22,350 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:22,367 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:22,371 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:22,386 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:27:22,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5}] 2023-10-02T12:27:22,400 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:27:22,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:27:22,563 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:22,565 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:27:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:27:22,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/f386b79e96ec4765971a99bd5fb5a8b3 is 49, key is activesession:/session:c/1696249642272/Put/seqid=0 2023-10-02T12:27:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:27:23,101 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:23,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:27:23,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:23,128 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962496342532 2023-10-02T12:27:23,128 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962496342532 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:23,129 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962496342533 2023-10-02T12:27:23,129 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962496342533 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:23,130 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962496342531 2023-10-02T12:27:23,130 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962496342531 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:23,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:27:23,131 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:23,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249634253 2023-10-02T12:27:23,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696249634253 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:23,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/f386b79e96ec4765971a99bd5fb5a8b3 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:23,231 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:27:23,235 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 670ms, sequenceid=5, compaction requested=false 2023-10-02T12:27:23,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:27:23,236 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:23,237 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. for snapshot_backup_system completed. 2023-10-02T12:27:23,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:27:23,244 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:27:23,248 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:27:23,249 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3] hfiles 2023-10-02T12:27:23,249 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 for snapshot=snapshot_backup_system 2023-10-02T12:27:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:27:23,666 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:23,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:27:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:27:23,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:23,678 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:23,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:27:23,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 in 1.2890 sec 2023-10-02T12:27:23,685 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:27:23,687 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:23,693 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:23,693 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:27:23,696 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:27:23,734 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:23,751 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:23,752 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:27:23,755 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:23,755 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:27:23,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4250 sec 2023-10-02T12:27:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:27:24,454 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(115): Backup backup_1696249642179 started at 1696249644454. 2023-10-02T12:27:24,476 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(119): Backup session backup_1696249642179 has been started. Backup session finished. Status: FAILURE 2023-10-02T12:27:24,477 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:24,477 ERROR [Listener at jenkins-hbase3.apache.org/38427 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] ... 32 more 2023-10-02T12:27:24,486 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371be1f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:24,493 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbbd1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:24,494 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:24,496 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45340, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:24,501 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:24,502 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1696249642179,startts=1696249644454,failedts=1696249644523,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1696249642179 2023-10-02T12:27:24,523 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:27:24,525 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:27:24,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:27:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:27:24,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249644557"}]},"ts":"1696249644557"} 2023-10-02T12:27:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:27:24,559 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:27:24,561 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:27:24,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN}] 2023-10-02T12:27:24,565 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN 2023-10-02T12:27:24,566 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:24,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:27:24,730 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(119): Close bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:24,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1668): Closing bca8be4589edc7e4cee153d1c0f19ab5, disabling compactions & flushes 2023-10-02T12:27:24,730 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1690): Closing region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:24,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:24,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. after waiting 0 ms 2023-10-02T12:27:24,730 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:24,730 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=214 B heapSize=800 B 2023-10-02T12:27:24,740 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/58b2669e174b4c84a06f07610e54348c is 218, key is session:backup_1696249642179/session:context/1696249644475/Put/seqid=0 2023-10-02T12:27:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:27:25,151 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/58b2669e174b4c84a06f07610e54348c 2023-10-02T12:27:25,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/58b2669e174b4c84a06f07610e54348c as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/58b2669e174b4c84a06f07610e54348c 2023-10-02T12:27:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:27:25,171 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/58b2669e174b4c84a06f07610e54348c, entries=1, sequenceid=9, filesize=5.2 K 2023-10-02T12:27:25,173 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 443ms, sequenceid=9, compaction requested=false 2023-10-02T12:27:25,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-10-02T12:27:25,188 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:25,189 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1909): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:25,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1622): Region close journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:25,191 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(163): Closed bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:25,192 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSED 2023-10-02T12:27:25,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-10-02T12:27:25,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 626 msec 2023-10-02T12:27:25,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:27:25,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN in 633 msec 2023-10-02T12:27:25,199 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249645199"}]},"ts":"1696249645199"} 2023-10-02T12:27:25,201 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:27:25,203 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:27:25,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; DisableTableProcedure table=backup:system in 657 msec 2023-10-02T12:27:25,392 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:27:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:27:25,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249645677 (current time:1696249645677). 2023-10-02T12:27:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696249645676 VERSION not specified, setting to 2 2023-10-02T12:27:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86e8d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:25,689 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37186, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-10-02T12:27:25,695 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:27:25,696 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:25,699 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249645676 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:25,701 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:27:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:27:26,112 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:27:26,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:27:26,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:27:26,115 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:26,115 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(327): No files under family: meta 2023-10-02T12:27:26,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:26,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/58b2669e174b4c84a06f07610e54348c, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3] hfiles 2023-10-02T12:27:26,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/58b2669e174b4c84a06f07610e54348c 2023-10-02T12:27:26,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:27:26,534 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:27:26,535 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:26,536 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:26,536 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696249645676 2023-10-02T12:27:26,537 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249645676 2023-10-02T12:27:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:27:26,964 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:26,989 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:26,990 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249645676 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696249645676 2023-10-02T12:27:26,994 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:26,994 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-10-02T12:27:26,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249645676 table=backup:system type=FLUSH ttl=0 } in 1.3030 sec 2023-10-02T12:27:27,123 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:27:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:27:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249642294 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:27:27,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:27:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:27:27,910 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-10-02T12:27:27,910 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-10-02T12:27:27,912 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:27,919 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249642294 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:27,920 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system 2023-10-02T12:27:27,925 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:27:27,925 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:27,938 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:27,958 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/58b2669e174b4c84a06f07610e54348c to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/58b2669e174b4c84a06f07610e54348c 2023-10-02T12:27:27,959 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:27,960 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249642294 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:27,962 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249647962"}]},"ts":"9223372036854775807"} 2023-10-02T12:27:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:27:28,043 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:27:28,043 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:28,044 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249647963"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249647963"}]},"ts":"1696249647963"} 2023-10-02T12:27:28,054 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:28,055 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:27:28,055 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:28,055 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:27:28,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249642294 type: FLUSH version: 2 ttl: 0 ) in 218 msec 2023-10-02T12:27:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:27:28,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249645676" 2023-10-02T12:27:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696249645676 2023-10-02T12:27:28,196 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:27:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:27:28,214 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:27:28,214 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249648214"}]},"ts":"1696249648214"} 2023-10-02T12:27:28,219 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:27:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:27:28,226 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:27:28,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN}] 2023-10-02T12:27:28,228 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN 2023-10-02T12:27:28,230 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:27:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:27:28,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:28,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:27:28,550 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:28,550 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:28,550 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:28,551 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,551 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:28,551 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,551 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,554 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,556 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName meta 2023-10-02T12:27:28,556 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:28,557 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:28,557 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,560 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName session 2023-10-02T12:27:28,560 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:28,599 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:28,600 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:28,602 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,608 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:27:28,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:28,628 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened bca8be4589edc7e4cee153d1c0f19ab5; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11187951520, jitterRate=0.04195918142795563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:27:28,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:28,630 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., pid=41, masterSystemTime=1696249648538 2023-10-02T12:27:28,633 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:28,633 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:28,634 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPEN, openSeqNum=13, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:28,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-10-02T12:27:28,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 251 msec 2023-10-02T12:27:28,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-10-02T12:27:28,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN in 413 msec 2023-10-02T12:27:28,643 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249648643"}]},"ts":"1696249648643"} 2023-10-02T12:27:28,645 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:27:28,647 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:27:28,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; EnableTableProcedure table=backup:system in 449 msec 2023-10-02T12:27:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:27:28,833 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:27:28,833 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:27:28,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:27:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:27:28,847 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:27:28,847 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:27:28,849 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1696249642179 2023-10-02T12:27:28,869 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:28,869 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:28,898 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupRepair(57): Running stage 1 2023-10-02T12:27:28,952 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1fe6a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:28,958 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:28,961 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45346, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:28,970 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:28,974 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37190, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:29,044 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:27:29,052 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249649052 (current time:1696249649052). 2023-10-02T12:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42fbfa43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:29,081 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37192, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:27:29,106 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:29,108 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:29,115 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:29,122 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:27:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:27:29,541 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:27:29,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5}] 2023-10-02T12:27:29,544 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:29,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-10-02T12:27:29,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:29,698 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-10-02T12:27:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:27:29,715 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/00bb5904b96c40b38015577db414a571 is 284, key is session:backup_1696249642179/session:context/1696249648848/Put/seqid=0 2023-10-02T12:27:29,731 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:29,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/00bb5904b96c40b38015577db414a571 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:29,774 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571, entries=2, sequenceid=18, filesize=5.2 K 2023-10-02T12:27:29,777 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 79ms, sequenceid=18, compaction requested=false 2023-10-02T12:27:29,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:29,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. for snapshot_backup_system completed. 2023-10-02T12:27:29,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:27:29,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:27:29,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:27:29,777 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571] hfiles 2023-10-02T12:27:29,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 for snapshot=snapshot_backup_system 2023-10-02T12:27:29,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 for snapshot=snapshot_backup_system 2023-10-02T12:27:30,200 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:30,200 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-10-02T12:27:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-10-02T12:27:30,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:30,211 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:27:30,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-10-02T12:27:30,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 in 672 msec 2023-10-02T12:27:30,221 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:27:30,222 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:30,223 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:30,223 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:27:30,225 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:27:30,279 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:30,302 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:30,304 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:27:30,307 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:30,307 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:27:30,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.2190 sec 2023-10-02T12:27:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:27:31,216 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(115): Backup backup_1696249649016 started at 1696249651216. 2023-10-02T12:27:31,219 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(119): Backup session backup_1696249649016 has been started. Backup session finished. Status: FAILURE 2023-10-02T12:27:31,223 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:31,224 ERROR [Listener at jenkins-hbase3.apache.org/38427 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] ... 32 more 2023-10-02T12:27:31,245 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2353f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:31,256 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36104a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:31,339 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:31,341 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50890, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:31,348 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:31,350 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42160, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1696249649016,startts=1696249651216,failedts=1696249651379,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1696249649016 2023-10-02T12:27:31,379 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:27:31,380 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:27:31,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:27:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:27:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:27:31,394 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249651394"}]},"ts":"1696249651394"} 2023-10-02T12:27:31,396 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:27:31,399 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:27:31,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN}] 2023-10-02T12:27:31,402 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN 2023-10-02T12:27:31,403 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:31,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:27:31,557 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:31,558 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing bca8be4589edc7e4cee153d1c0f19ab5, disabling compactions & flushes 2023-10-02T12:27:31,558 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:31,558 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:31,558 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. after waiting 0 ms 2023-10-02T12:27:31,558 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:31,558 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=289 B heapSize=952 B 2023-10-02T12:27:31,583 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/49f549db0be64c96b133cbb798fbc64a is 79, key is startcode:hdfs://localhost:34623/backupUT/meta:startcode/1696249651221/Put/seqid=0 2023-10-02T12:27:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:27:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:27:32,018 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=75 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/49f549db0be64c96b133cbb798fbc64a 2023-10-02T12:27:32,037 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/c42451a173054752b1959ae9684efc04 is 218, key is session:backup_1696249649016/session:context/1696249651217/Put/seqid=0 2023-10-02T12:27:32,053 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/c42451a173054752b1959ae9684efc04 2023-10-02T12:27:32,063 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/49f549db0be64c96b133cbb798fbc64a as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/49f549db0be64c96b133cbb798fbc64a 2023-10-02T12:27:32,072 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/49f549db0be64c96b133cbb798fbc64a, entries=1, sequenceid=23, filesize=5.1 K 2023-10-02T12:27:32,074 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/c42451a173054752b1959ae9684efc04 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/c42451a173054752b1959ae9684efc04 2023-10-02T12:27:32,088 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/c42451a173054752b1959ae9684efc04, entries=1, sequenceid=23, filesize=5.2 K 2023-10-02T12:27:32,089 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(3022): Finished flush of dataSize ~289 B/289, heapSize ~920 B/920, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 531ms, sequenceid=23, compaction requested=true 2023-10-02T12:27:32,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=12 2023-10-02T12:27:32,125 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:32,125 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:32,125 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:32,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:32,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSED 2023-10-02T12:27:32,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-10-02T12:27:32,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 731 msec 2023-10-02T12:27:32,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-10-02T12:27:32,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN in 737 msec 2023-10-02T12:27:32,141 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249652141"}]},"ts":"1696249652141"} 2023-10-02T12:27:32,143 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:27:32,145 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:27:32,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=backup:system in 757 msec 2023-10-02T12:27:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:27:32,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249652521 (current time:1696249652521). 2023-10-02T12:27:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696249652519 VERSION not specified, setting to 2 2023-10-02T12:27:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcfa7ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:32,544 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42164, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-10-02T12:27:32,566 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:27:32,568 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:32,572 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249652519 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:32,576 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:32,602 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:27:32,612 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:27:32,612 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:27:32,612 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:32,616 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/49f549db0be64c96b133cbb798fbc64a] hfiles 2023-10-02T12:27:32,616 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/49f549db0be64c96b133cbb798fbc64a 2023-10-02T12:27:32,616 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:32,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/c42451a173054752b1959ae9684efc04, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3] hfiles 2023-10-02T12:27:32,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:32,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/c42451a173054752b1959ae9684efc04 2023-10-02T12:27:32,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:27:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:27:33,048 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:27:33,050 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:33,051 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:33,051 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696249652519 2023-10-02T12:27:33,054 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249652519 2023-10-02T12:27:33,080 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:33,098 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:33,099 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249652519 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696249652519 2023-10-02T12:27:33,104 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:33,105 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-10-02T12:27:33,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249652519 table=backup:system type=FLUSH ttl=0 } in 548 msec 2023-10-02T12:27:33,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:27:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:27:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249649052 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:27:33,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:27:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:27:33,247 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-10-02T12:27:33,247 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-10-02T12:27:33,249 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:33,252 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249649052 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:33,252 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system 2023-10-02T12:27:33,256 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:27:33,256 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,277 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/49f549db0be64c96b133cbb798fbc64a to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/49f549db0be64c96b133cbb798fbc64a 2023-10-02T12:27:33,278 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:33,290 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/c42451a173054752b1959ae9684efc04 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/c42451a173054752b1959ae9684efc04 2023-10-02T12:27:33,290 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:33,291 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249649052 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:33,294 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249653294"}]},"ts":"9223372036854775807"} 2023-10-02T12:27:33,299 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:27:33,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:33,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249653295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249653295"}]},"ts":"1696249653295"} 2023-10-02T12:27:33,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:33,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:27:33,302 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:33,302 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:27:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:27:33,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249649052 type: FLUSH version: 2 ttl: 0 ) in 110 msec 2023-10-02T12:27:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:27:33,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249652519" 2023-10-02T12:27:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696249652519 2023-10-02T12:27:33,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:27:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:27:33,536 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:27:33,536 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249653536"}]},"ts":"1696249653536"} 2023-10-02T12:27:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:27:33,538 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:27:33,549 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:27:33,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN}] 2023-10-02T12:27:33,552 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN 2023-10-02T12:27:33,555 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:27:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:27:33,707 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:33,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:27:33,868 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(130): Open backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:33,868 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7254): Opening region: {ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:33,869 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:33,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(888): Instantiated backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:33,869 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7296): checking encryption for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,870 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7299): checking classloading for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,871 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,873 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName meta 2023-10-02T12:27:33,873 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:33,874 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:33,874 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,875 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName session 2023-10-02T12:27:33,875 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:33,887 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:33,898 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:33,898 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:33,899 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,901 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,903 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:27:33,905 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1076): writing seq id for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:33,906 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): Opened bca8be4589edc7e4cee153d1c0f19ab5; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10009023840, jitterRate=-0.06783701479434967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:27:33,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(992): Region open journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:33,908 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., pid=51, masterSystemTime=1696249653863 2023-10-02T12:27:33,910 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:33,911 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(158): Opened backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:33,911 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPEN, openSeqNum=27, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:33,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-10-02T12:27:33,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 203 msec 2023-10-02T12:27:33,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=50, resume processing ppid=49 2023-10-02T12:27:33,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, ppid=49, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN in 366 msec 2023-10-02T12:27:33,919 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249653919"}]},"ts":"1696249653919"} 2023-10-02T12:27:33,921 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:27:33,923 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:27:33,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; EnableTableProcedure table=backup:system in 396 msec 2023-10-02T12:27:34,056 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:27:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:27:34,144 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:27:34,144 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:27:34,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:27:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:27:34,157 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:27:34,157 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:27:34,161 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1696249649016 2023-10-02T12:27:34,168 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:34,168 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:34,242 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupRepair(57): Running stage 2 2023-10-02T12:27:34,306 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a8c3df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:34,310 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:34,313 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50892, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:34,317 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:34,319 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:34,421 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:27:34,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249654427 (current time:1696249654427). 2023-10-02T12:27:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:27:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58fc85f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:34,452 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42180, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:27:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:27:34,477 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:34,478 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:34,483 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:34,486 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:34,503 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:27:34,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5}] 2023-10-02T12:27:34,505 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:27:34,569 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:27:34,573 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50898, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:27:34,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-10-02T12:27:34,658 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:34,660 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-10-02T12:27:34,681 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/45dac6fbbd044adaaacf341feb02ab2f is 284, key is session:backup_1696249649016/session:context/1696249654158/Put/seqid=0 2023-10-02T12:27:34,690 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/45dac6fbbd044adaaacf341feb02ab2f 2023-10-02T12:27:34,704 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/45dac6fbbd044adaaacf341feb02ab2f as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f 2023-10-02T12:27:34,719 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f, entries=2, sequenceid=32, filesize=5.2 K 2023-10-02T12:27:34,721 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 61ms, sequenceid=32, compaction requested=true 2023-10-02T12:27:34,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:34,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. for snapshot_backup_system completed. 2023-10-02T12:27:34,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:27:34,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:27:34,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:27:34,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f] hfiles 2023-10-02T12:27:34,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 for snapshot=snapshot_backup_system 2023-10-02T12:27:34,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 for snapshot=snapshot_backup_system 2023-10-02T12:27:34,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f for snapshot=snapshot_backup_system 2023-10-02T12:27:34,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:34,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-10-02T12:27:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-10-02T12:27:34,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:34,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:34,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-10-02T12:27:34,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 in 249 msec 2023-10-02T12:27:34,754 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:27:34,757 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:34,758 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:34,758 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:27:34,759 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:27:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:27:34,825 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:34,846 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:34,847 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:27:34,850 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:34,850 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:27:34,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 390 msec 2023-10-02T12:27:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:27:35,073 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(115): Backup backup_1696249654387 started at 1696249655072. 2023-10-02T12:27:35,075 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(119): Backup session backup_1696249654387 has been started. 2023-10-02T12:27:35,078 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-10-02T12:27:35,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:27:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:27:35,109 INFO [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:27:35,109 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:27:35,110 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:27:35,110 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:27:35,114 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:27:35,116 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,126 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:27:35,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:27:35,126 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:27:35,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:27:35,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:27:35,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:27:35,129 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:27:35,132 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:27:35,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:27:35,133 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:27:35,133 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:27:35,135 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:27:35,135 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:27:35,135 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,33719,1696249616681' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:27:35,139 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:35,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:27:35,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:27:35,139 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:27:35,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:27:35,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:27:35,141 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:35,141 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,142 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:27:35,142 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,33719,1696249616681' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:27:35,143 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:27:35,143 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3428ae78[Count = 0] remaining members to acquire global barrier 2023-10-02T12:27:35,143 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:35,145 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:35,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:35,146 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:27:35,146 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:27:35,147 DEBUG [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,147 INFO [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249621718 highest: 1696249621718 on jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,148 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta:.meta(num 1696249622549) roll requested 2023-10-02T12:27:35,147 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:27:35,165 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249655148.meta, exclude list is [], retry=0 2023-10-02T12:27:35,172 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:27:35,176 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249622549.meta with entries=56, filesize=15.96 KB; new WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249655148.meta 2023-10-02T12:27:35,178 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:27:35,178 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249622549.meta is not closed yet, will try archiving it next time 2023-10-02T12:27:35,178 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C33719%2C1696249616681:(num 1696249621718) roll requested 2023-10-02T12:27:35,198 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249655178, exclude list is [], retry=0 2023-10-02T12:27:35,203 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:27:35,208 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249621718 with entries=40, filesize=9.34 KB; new WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249655178 2023-10-02T12:27:35,209 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:27:35,209 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249621718 is not closed yet, will try archiving it next time 2023-10-02T12:27:35,215 DEBUG [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:27:35,215 INFO [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249655178 on jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,217 DEBUG [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:35,219 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50906, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:27:35,244 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:27:35,244 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,33719,1696249616681' in zk 2023-10-02T12:27:35,250 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:27:35,250 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,250 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:27:35,250 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:27:35,250 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:27:35,250 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:27:35,252 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:27:35,253 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:27:35,253 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:35,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:27:35,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:35,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,33719,1696249616681': 2023-10-02T12:27:35,256 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,33719,1696249616681' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:27:35,256 INFO [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:27:35,256 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:27:35,257 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:27:35,257 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:27:35,257 INFO [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:27:35,260 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:27:35,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:27:35,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:27:35,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:27:35,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:27:35,261 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:27:35,261 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:27:35,261 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:27:35,261 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:27:35,261 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:35,262 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:27:35,263 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:35,263 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:27:35,264 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:35,265 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,277 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:27:35,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:27:35,277 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:27:35,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:27:35,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:27:35,279 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:27:35,279 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:27:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done Backup session finished. Status: FAILURE 2023-10-02T12:27:35,393 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:35,393 ERROR [Listener at jenkins-hbase3.apache.org/38427 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] ... 32 more 2023-10-02T12:27:35,403 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9445d72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:35,413 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6f8453, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:35,414 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:35,417 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50922, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:35,426 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:35,429 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1696249654387,startts=1696249655072,failedts=1696249655459,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1696249654387 2023-10-02T12:27:35,459 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:27:35,460 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:27:35,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:27:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:27:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:27:35,478 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249655478"}]},"ts":"1696249655478"} 2023-10-02T12:27:35,479 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:27:35,489 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:27:35,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN}] 2023-10-02T12:27:35,494 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN 2023-10-02T12:27:35,495 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:35,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:27:35,665 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:35,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing bca8be4589edc7e4cee153d1c0f19ab5, disabling compactions & flushes 2023-10-02T12:27:35,665 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:35,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:35,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. after waiting 0 ms 2023-10-02T12:27:35,665 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:35,665 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-10-02T12:27:35,688 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/5a449620e4694909b9cda89397575533 is 116, key is rslogts:hdfs://localhost:34623/backupUT\x00jenkins-hbase3.apache.org:33719/meta:rs-log-ts/1696249655242/Put/seqid=0 2023-10-02T12:27:35,704 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/5a449620e4694909b9cda89397575533 2023-10-02T12:27:35,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/fbcdfa0dcf754e66bb1c715f1ae5a46b is 218, key is session:backup_1696249654387/session:context/1696249655074/Put/seqid=0 2023-10-02T12:27:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:27:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:27:36,127 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/fbcdfa0dcf754e66bb1c715f1ae5a46b 2023-10-02T12:27:36,139 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/5a449620e4694909b9cda89397575533 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/5a449620e4694909b9cda89397575533 2023-10-02T12:27:36,148 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/5a449620e4694909b9cda89397575533, entries=2, sequenceid=38, filesize=5.3 K 2023-10-02T12:27:36,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/fbcdfa0dcf754e66bb1c715f1ae5a46b as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/fbcdfa0dcf754e66bb1c715f1ae5a46b 2023-10-02T12:27:36,161 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/fbcdfa0dcf754e66bb1c715f1ae5a46b, entries=1, sequenceid=38, filesize=5.2 K 2023-10-02T12:27:36,162 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 497ms, sequenceid=38, compaction requested=true 2023-10-02T12:27:36,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/recovered.edits/41.seqid, newMaxSeqId=41, maxSeqId=26 2023-10-02T12:27:36,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:36,184 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:36,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:36,188 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:36,189 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSED 2023-10-02T12:27:36,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-10-02T12:27:36,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 687 msec 2023-10-02T12:27:36,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-10-02T12:27:36,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN in 701 msec 2023-10-02T12:27:36,196 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249656195"}]},"ts":"1696249656195"} 2023-10-02T12:27:36,197 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:27:36,199 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:27:36,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=backup:system in 735 msec 2023-10-02T12:27:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:27:36,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249656597 (current time:1696249656597). 2023-10-02T12:27:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696249656596 VERSION not specified, setting to 2 2023-10-02T12:27:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e105b3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:36,617 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42202, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-10-02T12:27:36,624 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:27:36,626 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:36,636 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249656596 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:36,639 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:27:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:27:37,055 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:27:37,056 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:27:37,056 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:27:37,057 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:37,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/5a449620e4694909b9cda89397575533] hfiles 2023-10-02T12:27:37,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/5a449620e4694909b9cda89397575533 2023-10-02T12:27:37,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:37,061 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/fbcdfa0dcf754e66bb1c715f1ae5a46b] hfiles 2023-10-02T12:27:37,061 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:37,061 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f 2023-10-02T12:27:37,061 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:37,061 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/fbcdfa0dcf754e66bb1c715f1ae5a46b 2023-10-02T12:27:37,077 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:27:37,078 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:37,079 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:37,079 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696249656596 2023-10-02T12:27:37,080 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249656596 2023-10-02T12:27:37,099 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:37,115 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:37,116 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249656596 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696249656596 2023-10-02T12:27:37,118 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:37,118 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-10-02T12:27:37,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249656596 table=backup:system type=FLUSH ttl=0 } in 500 msec 2023-10-02T12:27:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:27:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249654427 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:27:37,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:27:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:27:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:27:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:27:37,663 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-10-02T12:27:37,663 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-10-02T12:27:37,664 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:37,668 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249654427 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:37,668 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system 2023-10-02T12:27:37,672 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:27:37,672 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:37,696 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/5a449620e4694909b9cda89397575533 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/5a449620e4694909b9cda89397575533 2023-10-02T12:27:37,696 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:37,707 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/fbcdfa0dcf754e66bb1c715f1ae5a46b to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/fbcdfa0dcf754e66bb1c715f1ae5a46b 2023-10-02T12:27:37,707 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:37,707 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249654427 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:37,709 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249657708"}]},"ts":"9223372036854775807"} 2023-10-02T12:27:37,712 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:27:37,712 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:37,713 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249657709"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249657709"}]},"ts":"1696249657709"} 2023-10-02T12:27:37,715 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:37,715 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:27:37,715 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:37,716 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:27:37,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249654427 type: FLUSH version: 2 ttl: 0 ) in 478 msec 2023-10-02T12:27:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:27:37,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249656596" 2023-10-02T12:27:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696249656596 2023-10-02T12:27:37,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:27:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:27:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:27:37,866 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:27:37,866 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249657866"}]},"ts":"1696249657866"} 2023-10-02T12:27:37,869 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:27:37,934 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:27:37,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN}] 2023-10-02T12:27:37,937 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN 2023-10-02T12:27:37,938 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:27:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:27:38,090 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:38,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:27:38,250 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(130): Open backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:38,250 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7254): Opening region: {ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:38,251 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:38,251 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,251 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(888): Instantiated backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:38,251 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7296): checking encryption for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,251 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7299): checking classloading for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,252 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,253 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName meta 2023-10-02T12:27:38,254 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:38,254 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:38,254 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,255 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName session 2023-10-02T12:27:38,255 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:38,269 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:38,278 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f 2023-10-02T12:27:38,287 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:38,287 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:38,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,290 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,292 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:27:38,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1076): writing seq id for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:38,298 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): Opened bca8be4589edc7e4cee153d1c0f19ab5; next sequenceid=42; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9945343840, jitterRate=-0.07376767694950104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:27:38,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(992): Region open journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:38,300 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., pid=61, masterSystemTime=1696249658244 2023-10-02T12:27:38,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(403): Add compact mark for store bca8be4589edc7e4cee153d1c0f19ab5:session, priority=-2147483648, current under compaction store size is 1 2023-10-02T12:27:38,307 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-10-02T12:27:38,307 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-10-02T12:27:38,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:38,309 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(158): Opened backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:38,311 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPEN, openSeqNum=42, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:38,312 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 15770 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-10-02T12:27:38,317 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.HStore(1534): bca8be4589edc7e4cee153d1c0f19ab5/session is initiating minor compaction (all files) 2023-10-02T12:27:38,318 INFO [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of bca8be4589edc7e4cee153d1c0f19ab5/session in backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:38,318 INFO [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f] into tmpdir=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp, totalSize=15.4 K 2023-10-02T12:27:38,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-10-02T12:27:38,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 224 msec 2023-10-02T12:27:38,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=60, resume processing ppid=59 2023-10-02T12:27:38,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, ppid=59, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN in 387 msec 2023-10-02T12:27:38,323 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] compactions.Compactor(223): Compacting f386b79e96ec4765971a99bd5fb5a8b3, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1696249642272 2023-10-02T12:27:38,324 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] compactions.Compactor(223): Compacting 00bb5904b96c40b38015577db414a571, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1696249648848 2023-10-02T12:27:38,324 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249658324"}]},"ts":"1696249658324"} 2023-10-02T12:27:38,325 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] compactions.Compactor(223): Compacting 45dac6fbbd044adaaacf341feb02ab2f, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1696249654158 2023-10-02T12:27:38,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:27:38,367 INFO [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bca8be4589edc7e4cee153d1c0f19ab5#session#compaction#8 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-10-02T12:27:38,368 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/219aa0a953d54a51a721c1f72416d8f4 is 284, key is session:backup_1696249642179/session:context/1696249648848/Put/seqid=0 2023-10-02T12:27:38,379 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:27:38,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; EnableTableProcedure table=backup:system in 520 msec 2023-10-02T12:27:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:27:38,469 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:27:38,469 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:27:38,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:27:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:27:38,487 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:27:38,487 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:27:38,489 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1696249654387 2023-10-02T12:27:38,496 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:38,496 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:38,559 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupRepair(57): Running stage 3 2023-10-02T12:27:38,734 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77514e22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:38,735 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:38,738 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50924, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:38,747 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:38,751 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42214, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:38,788 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/219aa0a953d54a51a721c1f72416d8f4 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/219aa0a953d54a51a721c1f72416d8f4 2023-10-02T12:27:38,811 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:27:38,812 INFO [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in bca8be4589edc7e4cee153d1c0f19ab5/session of bca8be4589edc7e4cee153d1c0f19ab5 into 219aa0a953d54a51a721c1f72416d8f4(size=5.6 K), total size for store is 5.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-10-02T12:27:38,812 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:38,812 INFO [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., storeName=bca8be4589edc7e4cee153d1c0f19ab5/session, priority=13, startTime=1696249658303; duration=0sec 2023-10-02T12:27:38,813 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-10-02T12:27:38,813 DEBUG [RS:0;jenkins-hbase3:33719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bca8be4589edc7e4cee153d1c0f19ab5:session 2023-10-02T12:27:38,814 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:27:38,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249658824 (current time:1696249658824). 2023-10-02T12:27:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:27:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ca977b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:38,899 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42216, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:27:38,904 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:27:38,905 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:38,909 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:38,911 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:38,924 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:27:38,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5}] 2023-10-02T12:27:38,925 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:27:39,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-10-02T12:27:39,077 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:39,078 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-10-02T12:27:39,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/5326b7190aac42159ef7b0a1f318e33f is 284, key is session:backup_1696249654387/session:context/1696249658488/Put/seqid=0 2023-10-02T12:27:39,093 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/5326b7190aac42159ef7b0a1f318e33f 2023-10-02T12:27:39,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/5326b7190aac42159ef7b0a1f318e33f as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f 2023-10-02T12:27:39,116 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f, entries=2, sequenceid=48, filesize=5.2 K 2023-10-02T12:27:39,117 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 39ms, sequenceid=48, compaction requested=false 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. for snapshot_backup_system completed. 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/219aa0a953d54a51a721c1f72416d8f4, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f] hfiles 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/219aa0a953d54a51a721c1f72416d8f4 for snapshot=snapshot_backup_system 2023-10-02T12:27:39,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f for snapshot=snapshot_backup_system 2023-10-02T12:27:39,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:39,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-10-02T12:27:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-10-02T12:27:39,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:39,139 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:39,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-10-02T12:27:39,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5 in 216 msec 2023-10-02T12:27:39,141 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:27:39,142 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:39,143 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:39,143 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:27:39,144 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:27:39,156 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:39,165 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:39,166 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:27:39,167 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:39,168 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:27:39,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 267 msec 2023-10-02T12:27:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:27:39,208 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(115): Backup backup_1696249658793 started at 1696249659208. 2023-10-02T12:27:39,210 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(119): Backup session backup_1696249658793 has been started. 2023-10-02T12:27:39,212 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-10-02T12:27:39,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:27:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:27:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:27:39,214 INFO [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:27:39,214 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:27:39,216 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:27:39,216 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:27:39,218 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:27:39,220 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:27:39,220 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:27:39,221 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:27:39,221 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:27:39,222 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:27:39,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:27:39,222 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:27:39,222 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:27:39,222 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:27:39,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:27:39,222 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:27:39,222 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:27:39,224 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:27:39,224 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:27:39,224 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,33719,1696249616681' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:27:39,227 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:39,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:27:39,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:27:39,228 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:27:39,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:27:39,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:27:39,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:39,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:27:39,230 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,33719,1696249616681' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:27:39,230 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6e6019b9[Count = 0] remaining members to acquire global barrier 2023-10-02T12:27:39,230 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:27:39,230 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:39,232 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:39,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:27:39,233 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:27:39,234 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:27:39,242 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:27:39,242 DEBUG [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,242 INFO [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249655178 highest: 1696249655178 on jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,242 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta:.meta(num 1696249655148) roll requested 2023-10-02T12:27:39,258 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249659242.meta, exclude list is [], retry=0 2023-10-02T12:27:39,267 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:27:39,270 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249655148.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249659242.meta 2023-10-02T12:27:39,270 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:27:39,271 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249655148.meta is not closed yet, will try archiving it next time 2023-10-02T12:27:39,271 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C33719%2C1696249616681:(num 1696249655178) roll requested 2023-10-02T12:27:39,302 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249655148.meta not finished, retry = 0 2023-10-02T12:27:39,309 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249659271, exclude list is [], retry=0 2023-10-02T12:27:39,319 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK] 2023-10-02T12:27:39,326 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249655178 with entries=13, filesize=3.42 KB; new WAL /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249659271 2023-10-02T12:27:39,327 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40443,DS-ca5e1ca7-0c62-44ab-840d-f2b6ee79e620,DISK]] 2023-10-02T12:27:39,327 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.1696249655178 is not closed yet, will try archiving it next time 2023-10-02T12:27:39,332 DEBUG [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 90 2023-10-02T12:27:39,332 INFO [rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249659271 on jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,346 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:27:39,346 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,33719,1696249616681' in zk 2023-10-02T12:27:39,368 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:27:39,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,368 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:27:39,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:27:39,368 DEBUG [member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:27:39,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:27:39,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:27:39,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:27:39,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:39,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:27:39,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:39,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,375 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:27:39,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,33719,1696249616681': 2023-10-02T12:27:39,376 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,33719,1696249616681' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:27:39,377 INFO [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:27:39,377 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:27:39,377 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:27:39,377 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:27:39,377 INFO [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:27:39,379 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:27:39,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:27:39,380 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:27:39,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:27:39,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:27:39,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:27:39,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:27:39,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:27:39,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:27:39,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:39,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:27:39,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:39,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:27:39,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:27:39,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:39,399 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:27:39,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:27:39,400 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:27:39,400 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:27:39,400 DEBUG [(jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:27:39,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:27:39,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:27:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:27:39,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } 2023-10-02T12:27:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249659509 (current time:1696249659510). 2023-10-02T12:27:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249659508_default_test-1696249634253 VERSION not specified, setting to 2 2023-10-02T12:27:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206ca4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:39,539 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42232, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } 2023-10-02T12:27:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:27:39,545 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:39,546 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:27:39,550 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_1696249659508_default_test-1696249634253 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:39,554 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:39,571 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:27:39,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure a863321cd7a76d54b557bc7d0d9b8156}] 2023-10-02T12:27:39,575 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:27:39,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-10-02T12:27:39,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:39,729 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing a863321cd7a76d54b557bc7d0d9b8156 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:27:39,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/.tmp/f/82be4d34672e4d3a926d0608543d4eb0 is 37, key is row10/f:q1/1696249637223/Put/seqid=0 2023-10-02T12:27:39,759 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/.tmp/f/82be4d34672e4d3a926d0608543d4eb0 2023-10-02T12:27:39,767 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/.tmp/f/82be4d34672e4d3a926d0608543d4eb0 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/f/82be4d34672e4d3a926d0608543d4eb0 2023-10-02T12:27:39,780 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/f/82be4d34672e4d3a926d0608543d4eb0, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:27:39,782 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for a863321cd7a76d54b557bc7d0d9b8156 in 52ms, sequenceid=103, compaction requested=false 2023-10-02T12:27:39,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for a863321cd7a76d54b557bc7d0d9b8156: 2023-10-02T12:27:39,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. for snapshot_1696249659508_default_test-1696249634253 completed. 2023-10-02T12:27:39,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156.' region-info for snapshot=snapshot_1696249659508_default_test-1696249634253 2023-10-02T12:27:39,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:27:39,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/f/82be4d34672e4d3a926d0608543d4eb0] hfiles 2023-10-02T12:27:39,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/f/82be4d34672e4d3a926d0608543d4eb0 for snapshot=snapshot_1696249659508_default_test-1696249634253 2023-10-02T12:27:39,793 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:39,794 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-10-02T12:27:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-10-02T12:27:39,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249659508_default_test-1696249634253 on region a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:39,796 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:39,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-10-02T12:27:39,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure a863321cd7a76d54b557bc7d0d9b8156 in 225 msec 2023-10-02T12:27:39,799 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:27:39,800 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:39,801 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:39,801 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249659508_default_test-1696249634253 2023-10-02T12:27:39,802 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_1696249659508_default_test-1696249634253 2023-10-02T12:27:39,831 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:39,846 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:39,848 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_1696249659508_default_test-1696249634253 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/snapshot_1696249659508_default_test-1696249634253 2023-10-02T12:27:39,851 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:39,851 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:27:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:27:39,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696249659508_default_test-1696249634253 table=test-1696249634253 type=FLUSH ttl=0 } in 310 msec 2023-10-02T12:27:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:27:40,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } 2023-10-02T12:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249660155 (current time:1696249660155). 2023-10-02T12:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249660154_ns2_test-16962496342531 VERSION not specified, setting to 2 2023-10-02T12:27:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@adfb318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:40,174 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44456, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } 2023-10-02T12:27:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-10-02T12:27:40,181 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:27:40,182 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:40,186 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_1696249660154_ns2_test-16962496342531 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:40,190 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:27:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:27:40,599 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:27:40,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 2694c02ee78bcedb7abe52d8a1144cf9}] 2023-10-02T12:27:40,605 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:40,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-10-02T12:27:40,757 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:40,759 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing 2694c02ee78bcedb7abe52d8a1144cf9 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:27:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:27:40,790 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/.tmp/f/5e87d0f4f22646869951b95f5bba2ce1 is 37, key is row10/f:q1/1696249638058/Put/seqid=0 2023-10-02T12:27:41,197 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/.tmp/f/5e87d0f4f22646869951b95f5bba2ce1 2023-10-02T12:27:41,206 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/.tmp/f/5e87d0f4f22646869951b95f5bba2ce1 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/f/5e87d0f4f22646869951b95f5bba2ce1 2023-10-02T12:27:41,214 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/f/5e87d0f4f22646869951b95f5bba2ce1, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:27:41,215 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 2694c02ee78bcedb7abe52d8a1144cf9 in 456ms, sequenceid=103, compaction requested=false 2023-10-02T12:27:41,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 2694c02ee78bcedb7abe52d8a1144cf9: 2023-10-02T12:27:41,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. for snapshot_1696249660154_ns2_test-16962496342531 completed. 2023-10-02T12:27:41,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9.' region-info for snapshot=snapshot_1696249660154_ns2_test-16962496342531 2023-10-02T12:27:41,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:27:41,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/f/5e87d0f4f22646869951b95f5bba2ce1] hfiles 2023-10-02T12:27:41,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/f/5e87d0f4f22646869951b95f5bba2ce1 for snapshot=snapshot_1696249660154_ns2_test-16962496342531 2023-10-02T12:27:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:27:41,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:41,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-10-02T12:27:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster(3995): Remote procedure done, pid=67 2023-10-02T12:27:41,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249660154_ns2_test-16962496342531 on region 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:41,653 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:41,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-10-02T12:27:41,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure 2694c02ee78bcedb7abe52d8a1144cf9 in 1.0550 sec 2023-10-02T12:27:41,657 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:27:41,657 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:41,658 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:41,658 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249660154_ns2_test-16962496342531 2023-10-02T12:27:41,659 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_1696249660154_ns2_test-16962496342531 2023-10-02T12:27:41,731 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:41,746 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:41,747 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/snapshot_1696249660154_ns2_test-16962496342531 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/snapshot_1696249660154_ns2_test-16962496342531 2023-10-02T12:27:41,749 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:41,749 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-10-02T12:27:41,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249660154_ns2_test-16962496342531 table=ns2:test-16962496342531 type=FLUSH ttl=0 } in 1.5730 sec 2023-10-02T12:27:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 Backup session finished. Status: FAILURE 2023-10-02T12:27:42,297 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:42,297 ERROR [Listener at jenkins-hbase3.apache.org/38427 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] ... 32 more 2023-10-02T12:27:42,325 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606f7531, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:42,388 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60056c01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:42,411 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:27:42,413 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39500, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:27:42,452 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:42,456 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44458, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1696249658793,startts=1696249659208,failedts=1696249662510,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1696249658793 2023-10-02T12:27:42,510 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:27:42,514 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:27:42,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:27:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:27:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:27:42,591 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249662590"}]},"ts":"1696249662590"} 2023-10-02T12:27:42,614 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:27:42,622 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:27:42,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN}] 2023-10-02T12:27:42,628 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN 2023-10-02T12:27:42,634 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:42,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:27:42,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:42,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing bca8be4589edc7e4cee153d1c0f19ab5, disabling compactions & flushes 2023-10-02T12:27:42,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:42,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:42,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. after waiting 0 ms 2023-10-02T12:27:42,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:42,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-10-02T12:27:42,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/b2138a5bba4a4ed28afa45fba46e29c7 is 116, key is rslogts:hdfs://localhost:34623/backupUT\x00jenkins-hbase3.apache.org:33719/meta:rs-log-ts/1696249659342/Put/seqid=0 2023-10-02T12:27:42,873 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/b2138a5bba4a4ed28afa45fba46e29c7 2023-10-02T12:27:42,889 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/a9bbef5c70f142e3b3252a98650b23d6 is 218, key is session:backup_1696249658793/session:context/1696249659209/Put/seqid=0 2023-10-02T12:27:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:27:42,947 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/a9bbef5c70f142e3b3252a98650b23d6 2023-10-02T12:27:42,974 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/meta/b2138a5bba4a4ed28afa45fba46e29c7 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/b2138a5bba4a4ed28afa45fba46e29c7 2023-10-02T12:27:43,003 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/b2138a5bba4a4ed28afa45fba46e29c7, entries=2, sequenceid=54, filesize=5.3 K 2023-10-02T12:27:43,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/a9bbef5c70f142e3b3252a98650b23d6 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/a9bbef5c70f142e3b3252a98650b23d6 2023-10-02T12:27:43,033 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/a9bbef5c70f142e3b3252a98650b23d6, entries=1, sequenceid=54, filesize=5.2 K 2023-10-02T12:27:43,039 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 234ms, sequenceid=54, compaction requested=true 2023-10-02T12:27:43,067 DEBUG [StoreCloser-backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f] to archive 2023-10-02T12:27:43,076 DEBUG [StoreCloser-backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-10-02T12:27:43,103 DEBUG [StoreCloser-backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/f386b79e96ec4765971a99bd5fb5a8b3 2023-10-02T12:27:43,123 DEBUG [StoreCloser-backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/00bb5904b96c40b38015577db414a571 2023-10-02T12:27:43,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:27:43,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:27:43,134 DEBUG [StoreCloser-backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/45dac6fbbd044adaaacf341feb02ab2f 2023-10-02T12:27:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:27:43,310 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/recovered.edits/57.seqid, newMaxSeqId=57, maxSeqId=41 2023-10-02T12:27:43,319 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:43,320 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:43,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:43,323 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:43,325 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=CLOSED 2023-10-02T12:27:43,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-10-02T12:27:43,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 702 msec 2023-10-02T12:27:43,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-10-02T12:27:43,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, UNASSIGN in 725 msec 2023-10-02T12:27:43,358 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249663358"}]},"ts":"1696249663358"} 2023-10-02T12:27:43,361 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:27:43,363 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:27:43,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; DisableTableProcedure table=backup:system in 791 msec 2023-10-02T12:27:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:27:43,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249663738 (current time:1696249663738). 2023-10-02T12:27:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:27:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696249663735 VERSION not specified, setting to 2 2023-10-02T12:27:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c53e322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:27:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:27:43,773 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44474, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:27:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:27:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-10-02T12:27:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:27:43,798 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:27:43,800 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:27:43,830 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot, current working dir path: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249663735 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:27:43,834 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:27:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:27:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:27:44,250 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:27:44,266 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:27:44,266 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:27:44,266 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:44,273 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/b2138a5bba4a4ed28afa45fba46e29c7] hfiles 2023-10-02T12:27:44,273 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/b2138a5bba4a4ed28afa45fba46e29c7 2023-10-02T12:27:44,273 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:44,281 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/219aa0a953d54a51a721c1f72416d8f4, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/a9bbef5c70f142e3b3252a98650b23d6] hfiles 2023-10-02T12:27:44,281 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/219aa0a953d54a51a721c1f72416d8f4 2023-10-02T12:27:44,281 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f 2023-10-02T12:27:44,281 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/a9bbef5c70f142e3b3252a98650b23d6 2023-10-02T12:27:44,315 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:27:44,319 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:27:44,320 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:27:44,320 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696249663735 2023-10-02T12:27:44,321 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249663735 2023-10-02T12:27:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:27:44,760 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:27:44,807 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:27:44,810 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249663735 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696249663735 2023-10-02T12:27:44,819 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:27:44,819 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-10-02T12:27:44,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249663735 table=backup:system type=FLUSH ttl=0 } in 1.0440 sec 2023-10-02T12:27:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:27:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249658824 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:27:44,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:27:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:27:45,002 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-10-02T12:27:45,002 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-10-02T12:27:45,006 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:27:45,011 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249658824 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:45,011 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system 2023-10-02T12:27:45,016 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:27:45,016 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:27:45,052 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/b2138a5bba4a4ed28afa45fba46e29c7 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/meta/b2138a5bba4a4ed28afa45fba46e29c7 2023-10-02T12:27:45,053 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:45,086 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/a9bbef5c70f142e3b3252a98650b23d6 to hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/archive/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/a9bbef5c70f142e3b3252a98650b23d6 2023-10-02T12:27:45,087 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:45,087 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249658824 type: FLUSH version: 2 ttl: 0 2023-10-02T12:27:45,091 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249665090"}]},"ts":"9223372036854775807"} 2023-10-02T12:27:45,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:27:45,114 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:45,114 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249665091"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249665091"}]},"ts":"1696249665091"} 2023-10-02T12:27:45,138 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:27:45,138 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:27:45,138 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:27:45,138 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:27:45,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249658824 type: FLUSH version: 2 ttl: 0 ) in 202 msec 2023-10-02T12:27:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:27:45,258 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249663735" 2023-10-02T12:27:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696249663735 2023-10-02T12:27:45,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:27:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] procedure2.ProcedureExecutor(1032): Stored pid=73, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:27:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:27:45,287 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:27:45,287 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249665287"}]},"ts":"1696249665287"} 2023-10-02T12:27:45,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:27:45,331 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:27:45,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN}] 2023-10-02T12:27:45,334 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN 2023-10-02T12:27:45,343 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:27:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:27:45,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:45,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681}] 2023-10-02T12:27:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:27:45,710 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(130): Open backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:45,710 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7254): Opening region: {ENCODED => bca8be4589edc7e4cee153d1c0f19ab5, NAME => 'backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:27:45,710 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:27:45,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(888): Instantiated backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:27:45,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7296): checking encryption for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7299): checking classloading for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,734 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,746 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName meta 2023-10-02T12:27:45,747 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:45,758 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:45,758 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,766 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bca8be4589edc7e4cee153d1c0f19ab5 columnFamilyName session 2023-10-02T12:27:45,767 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:27:45,862 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/219aa0a953d54a51a721c1f72416d8f4 2023-10-02T12:27:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:27:45,920 DEBUG [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/5326b7190aac42159ef7b0a1f318e33f 2023-10-02T12:27:45,920 INFO [StoreOpener-bca8be4589edc7e4cee153d1c0f19ab5-1 {}] regionserver.HStore(324): Store=bca8be4589edc7e4cee153d1c0f19ab5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:27:45,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,932 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,934 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:27:45,939 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1076): writing seq id for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:45,940 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): Opened bca8be4589edc7e4cee153d1c0f19ab5; next sequenceid=58; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11154492480, jitterRate=0.0388430655002594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:27:45,941 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(992): Region open journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:45,951 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., pid=75, masterSystemTime=1696249665693 2023-10-02T12:27:45,958 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:45,959 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(158): Opened backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:45,962 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=bca8be4589edc7e4cee153d1c0f19ab5, regionState=OPEN, openSeqNum=58, regionLocation=jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:45,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-10-02T12:27:45,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; OpenRegionProcedure bca8be4589edc7e4cee153d1c0f19ab5, server=jenkins-hbase3.apache.org,33719,1696249616681 in 454 msec 2023-10-02T12:27:45,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-10-02T12:27:45,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bca8be4589edc7e4cee153d1c0f19ab5, ASSIGN in 642 msec 2023-10-02T12:27:45,976 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249665976"}]},"ts":"1696249665976"} 2023-10-02T12:27:45,978 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:27:45,990 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:27:45,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, state=SUCCESS; EnableTableProcedure table=backup:system in 713 msec 2023-10-02T12:27:46,042 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:27:46,042 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:27:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:27:46,411 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:27:46,412 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:27:46,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:27:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:27:46,450 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:27:46,450 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:27:46,460 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1696249658793 2023-10-02T12:27:46,466 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:46,466 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:46,937 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.ResourceChecker(175): after: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=602 (was 583) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_207596687_16 at /127.0.0.1:45696 [Receiving block BP-432441259-172.31.12.81-1696249606345:blk_1073741887_1063] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c07e9e3-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7fb7937b-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c07e9e3-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_207596687_16 at /127.0.0.1:45710 [Receiving block BP-432441259-172.31.12.81-1696249606345:blk_1073741888_1064] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c07e9e3-shared-pool-26 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,33719,1696249616681' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-432441259-172.31.12.81-1696249606345:blk_1073741888_1064, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c07e9e3-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1213520385_16 at /127.0.0.1:52454 [Waiting for operation #21] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,36809,1696249611352)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7fb7937b-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,33719,1696249616681)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1213520385_16 at /127.0.0.1:52512 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7fb7937b-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:33719-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1213520385_16 at /127.0.0.1:52496 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-432441259-172.31.12.81-1696249606345:blk_1073741887_1063, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7fb7937b-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c07e9e3-shared-pool-27 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c07e9e3-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7fb7937b-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=688 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=811 (was 763) - SystemLoadAverage LEAK? -, ProcessCount=181 (was 181), AvailableMemoryMB=2191 (was 1507) - AvailableMemoryMB LEAK? - 2023-10-02T12:27:46,938 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-10-02T12:27:46,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249659508_default_test-1696249634253" 2023-10-02T12:27:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249659508_default_test-1696249634253 2023-10-02T12:27:46,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249660154_ns2_test-16962496342531" 2023-10-02T12:27:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36809 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249660154_ns2_test-16962496342531 2023-10-02T12:27:47,053 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:27:47,058 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:47,059 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:27:47,059 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] util.JVMClusterUtil(257): Found active master hash=1648330127, stopped=false 2023-10-02T12:27:47,060 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:27:47,067 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:27:47,069 DEBUG [Listener at jenkins-hbase3.apache.org/38427 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:47,070 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,33719,1696249616681' ***** 2023-10-02T12:27:47,070 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:27:47,070 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:27:47,090 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:27:47,101 INFO [RS:0;jenkins-hbase3:33719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3b0c1ba4{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:27:47,126 INFO [RS:0;jenkins-hbase3:33719 {}] server.AbstractConnector(383): Stopped ServerConnector@33644794{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:27:47,126 INFO [RS:0;jenkins-hbase3:33719 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:27:47,128 INFO [RS:0;jenkins-hbase3:33719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@227fa1d8{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:27:47,131 INFO [RS:0;jenkins-hbase3:33719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4e44870a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.log.dir/,STOPPED} 2023-10-02T12:27:47,135 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:27:47,142 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:27:47,143 INFO [RS:0;jenkins-hbase3:33719 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:27:47,143 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:27:47,143 INFO [RS:0;jenkins-hbase3:33719 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:27:47,143 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for 60ae3f470b3b69f99aa07d4a8996765d 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for bca8be4589edc7e4cee153d1c0f19ab5 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for a863321cd7a76d54b557bc7d0d9b8156 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for 2694c02ee78bcedb7abe52d8a1144cf9 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for 55a233dd7aad88b70405150c81a513f0 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for 13109ef6404ab06943505340044ec89e 2023-10-02T12:27:47,144 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33719,1696249616681 2023-10-02T12:27:47,145 DEBUG [RS:0;jenkins-hbase3:33719 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:47,145 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 60ae3f470b3b69f99aa07d4a8996765d, disabling compactions & flushes 2023-10-02T12:27:47,145 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:47,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:47,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. after waiting 0 ms 2023-10-02T12:27:47,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:47,146 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:27:47,146 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:27:47,146 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:27:47,146 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:27:47,158 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-02T12:27:47,158 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1583): Online Regions={60ae3f470b3b69f99aa07d4a8996765d=backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d., bca8be4589edc7e4cee153d1c0f19ab5=backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5., a863321cd7a76d54b557bc7d0d9b8156=test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156., 1588230740=hbase:meta,,1.1588230740, c1335df7709a0befb7a49aa3f6cd3add=hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add., 2694c02ee78bcedb7abe52d8a1144cf9=ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9., 55a233dd7aad88b70405150c81a513f0=ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0., 13109ef6404ab06943505340044ec89e=ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e.} 2023-10-02T12:27:47,159 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1609): Waiting on 13109ef6404ab06943505340044ec89e, 1588230740, 2694c02ee78bcedb7abe52d8a1144cf9, 55a233dd7aad88b70405150c81a513f0, 60ae3f470b3b69f99aa07d4a8996765d, a863321cd7a76d54b557bc7d0d9b8156, bca8be4589edc7e4cee153d1c0f19ab5, c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:47,160 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:27:47,160 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:27:47,160 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:27:47,160 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:27:47,160 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:27:47,161 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=17.20 KB heapSize=30.02 KB 2023-10-02T12:27:47,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system_bulk/60ae3f470b3b69f99aa07d4a8996765d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:27:47,193 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,194 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:47,194 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 60ae3f470b3b69f99aa07d4a8996765d: 2023-10-02T12:27:47,194 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696249640831.60ae3f470b3b69f99aa07d4a8996765d. 2023-10-02T12:27:47,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing bca8be4589edc7e4cee153d1c0f19ab5, disabling compactions & flushes 2023-10-02T12:27:47,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:47,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:47,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. after waiting 0 ms 2023-10-02T12:27:47,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:47,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing bca8be4589edc7e4cee153d1c0f19ab5 2/2 column families, dataSize=324 B heapSize=984 B 2023-10-02T12:27:47,219 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/02671e4b62984cb68c04cbc82505ba3e is 284, key is session:backup_1696249658793/session:context/1696249666458/Put/seqid=0 2023-10-02T12:27:47,228 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/.tmp/info/cd0818f125a64c9dad26d5a1f9eb4596 is 159, key is ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9./info:regioninfo/1696249637922/Put/seqid=0 2023-10-02T12:27:47,270 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/02671e4b62984cb68c04cbc82505ba3e 2023-10-02T12:27:47,314 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/.tmp/session/02671e4b62984cb68c04cbc82505ba3e as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/02671e4b62984cb68c04cbc82505ba3e 2023-10-02T12:27:47,361 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1609): Waiting on 13109ef6404ab06943505340044ec89e, 1588230740, 2694c02ee78bcedb7abe52d8a1144cf9, 55a233dd7aad88b70405150c81a513f0, a863321cd7a76d54b557bc7d0d9b8156, bca8be4589edc7e4cee153d1c0f19ab5, c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:47,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/session/02671e4b62984cb68c04cbc82505ba3e, entries=2, sequenceid=62, filesize=5.2 K 2023-10-02T12:27:47,376 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~712 B/712, currentSize=0 B/0 for bca8be4589edc7e4cee153d1c0f19ab5 in 180ms, sequenceid=62, compaction requested=true 2023-10-02T12:27:47,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/backup/system/bca8be4589edc7e4cee153d1c0f19ab5/recovered.edits/65.seqid, newMaxSeqId=65, maxSeqId=57 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,440 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for bca8be4589edc7e4cee153d1c0f19ab5: 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696249639665.bca8be4589edc7e4cee153d1c0f19ab5. 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a863321cd7a76d54b557bc7d0d9b8156, disabling compactions & flushes 2023-10-02T12:27:47,440 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. after waiting 0 ms 2023-10-02T12:27:47,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:47,458 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:27:47,458 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:27:47,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/default/test-1696249634253/a863321cd7a76d54b557bc7d0d9b8156/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:27:47,498 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,502 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:47,502 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a863321cd7a76d54b557bc7d0d9b8156: 2023-10-02T12:27:47,502 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696249634253,,1696249634858.a863321cd7a76d54b557bc7d0d9b8156. 2023-10-02T12:27:47,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c1335df7709a0befb7a49aa3f6cd3add, disabling compactions & flushes 2023-10-02T12:27:47,506 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:47,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:47,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. after waiting 0 ms 2023-10-02T12:27:47,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:47,506 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing c1335df7709a0befb7a49aa3f6cd3add 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:27:47,556 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/.tmp/info/b4e7cd73e45a4c3e9353466e6d9b1183 is 45, key is default/info:d/1696249624843/Put/seqid=0 2023-10-02T12:27:47,561 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1609): Waiting on 13109ef6404ab06943505340044ec89e, 1588230740, 2694c02ee78bcedb7abe52d8a1144cf9, 55a233dd7aad88b70405150c81a513f0, c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:47,636 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/.tmp/info/b4e7cd73e45a4c3e9353466e6d9b1183 2023-10-02T12:27:47,651 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.80 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/.tmp/info/cd0818f125a64c9dad26d5a1f9eb4596 2023-10-02T12:27:47,666 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/.tmp/info/b4e7cd73e45a4c3e9353466e6d9b1183 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/info/b4e7cd73e45a4c3e9353466e6d9b1183 2023-10-02T12:27:47,677 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/info/b4e7cd73e45a4c3e9353466e6d9b1183, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:27:47,678 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for c1335df7709a0befb7a49aa3f6cd3add in 172ms, sequenceid=11, compaction requested=false 2023-10-02T12:27:47,711 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/.tmp/table/79defbc892734851bc4f01b0b3e7e6b8 is 59, key is ns2:test-16962496342531/table:state/1696249637936/Put/seqid=0 2023-10-02T12:27:47,761 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/namespace/c1335df7709a0befb7a49aa3f6cd3add/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:27:47,762 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1609): Waiting on 13109ef6404ab06943505340044ec89e, 1588230740, 2694c02ee78bcedb7abe52d8a1144cf9, 55a233dd7aad88b70405150c81a513f0, c1335df7709a0befb7a49aa3f6cd3add 2023-10-02T12:27:47,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,767 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:47,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c1335df7709a0befb7a49aa3f6cd3add: 2023-10-02T12:27:47,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696249623531.c1335df7709a0befb7a49aa3f6cd3add. 2023-10-02T12:27:47,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2694c02ee78bcedb7abe52d8a1144cf9, disabling compactions & flushes 2023-10-02T12:27:47,767 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:47,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:47,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. after waiting 0 ms 2023-10-02T12:27:47,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:47,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns2/test-16962496342531/2694c02ee78bcedb7abe52d8a1144cf9/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:27:47,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,778 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:47,778 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2694c02ee78bcedb7abe52d8a1144cf9: 2023-10-02T12:27:47,778 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962496342531,,1696249637415.2694c02ee78bcedb7abe52d8a1144cf9. 2023-10-02T12:27:47,779 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 55a233dd7aad88b70405150c81a513f0, disabling compactions & flushes 2023-10-02T12:27:47,780 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:47,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:47,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. after waiting 0 ms 2023-10-02T12:27:47,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:47,791 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns4/test-16962496342533/55a233dd7aad88b70405150c81a513f0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:27:47,792 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,793 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:47,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 55a233dd7aad88b70405150c81a513f0: 2023-10-02T12:27:47,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962496342533,,1696249638835.55a233dd7aad88b70405150c81a513f0. 2023-10-02T12:27:47,794 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 13109ef6404ab06943505340044ec89e, disabling compactions & flushes 2023-10-02T12:27:47,794 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:47,795 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:47,795 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. after waiting 0 ms 2023-10-02T12:27:47,795 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:47,809 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/ns3/test-16962496342532/13109ef6404ab06943505340044ec89e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:27:47,810 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:47,812 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:47,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 13109ef6404ab06943505340044ec89e: 2023-10-02T12:27:47,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962496342532,,1696249638201.13109ef6404ab06943505340044ec89e. 2023-10-02T12:27:47,966 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-10-02T12:27:48,155 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/.tmp/table/79defbc892734851bc4f01b0b3e7e6b8 2023-10-02T12:27:48,166 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-10-02T12:27:48,166 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740} 2023-10-02T12:27:48,166 DEBUG [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-10-02T12:27:48,168 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/.tmp/info/cd0818f125a64c9dad26d5a1f9eb4596 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/info/cd0818f125a64c9dad26d5a1f9eb4596 2023-10-02T12:27:48,178 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/info/cd0818f125a64c9dad26d5a1f9eb4596, entries=74, sequenceid=79, filesize=14.4 K 2023-10-02T12:27:48,180 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/.tmp/table/79defbc892734851bc4f01b0b3e7e6b8 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/table/79defbc892734851bc4f01b0b3e7e6b8 2023-10-02T12:27:48,187 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/table/79defbc892734851bc4f01b0b3e7e6b8, entries=15, sequenceid=79, filesize=5.8 K 2023-10-02T12:27:48,193 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~17.20 KB/17610, heapSize ~29.73 KB/30448, currentSize=0 B/0 for 1588230740 in 1033ms, sequenceid=79, compaction requested=false 2023-10-02T12:27:48,222 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=1 2023-10-02T12:27:48,230 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:27:48,230 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:27:48,231 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:27:48,231 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:27:48,231 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:27:48,367 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33719,1696249616681; all regions closed. 2023-10-02T12:27:48,387 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/WALs/jenkins-hbase3.apache.org,33719,1696249616681/jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta.1696249659242.meta not finished, retry = 0 2023-10-02T12:27:48,526 DEBUG [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/oldWALs 2023-10-02T12:27:48,526 INFO [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C33719%2C1696249616681.meta:.meta(num 1696249659242) 2023-10-02T12:27:48,544 DEBUG [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/oldWALs 2023-10-02T12:27:48,544 INFO [RS:0;jenkins-hbase3:33719 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C33719%2C1696249616681:(num 1696249659271) 2023-10-02T12:27:48,544 DEBUG [RS:0;jenkins-hbase3:33719 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:48,544 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:27:48,546 INFO [RS:0;jenkins-hbase3:33719 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-10-02T12:27:48,550 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:27:48,552 INFO [RS:0;jenkins-hbase3:33719 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:33719 2023-10-02T12:27:48,564 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,33719,1696249616681] 2023-10-02T12:27:48,565 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,33719,1696249616681; numProcessing=1 2023-10-02T12:27:48,570 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,33719,1696249616681 expired; onlineServers=0 2023-10-02T12:27:48,570 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,36809,1696249611352' ***** 2023-10-02T12:27:48,570 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:27:48,570 DEBUG [M:0;jenkins-hbase3:36809 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f691b7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:27:48,570 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:27:48,657 INFO [M:0;jenkins-hbase3:36809 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@150be744{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:27:48,665 INFO [M:0;jenkins-hbase3:36809 {}] server.AbstractConnector(383): Stopped ServerConnector@15a156df{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:27:48,665 INFO [M:0;jenkins-hbase3:36809 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:27:48,686 INFO [M:0;jenkins-hbase3:36809 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2fa0fe4a{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:27:48,687 INFO [M:0;jenkins-hbase3:36809 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7260be41{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/hadoop.log.dir/,STOPPED} 2023-10-02T12:27:48,690 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,36809,1696249611352 2023-10-02T12:27:48,690 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,36809,1696249611352; all regions closed. 2023-10-02T12:27:48,690 DEBUG [M:0;jenkins-hbase3:36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:48,690 INFO [M:0;jenkins-hbase3:36809 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:27:48,691 INFO [RS:0;jenkins-hbase3:33719 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33719,1696249616681; zookeeper connection closed. 2023-10-02T12:27:48,694 INFO [M:0;jenkins-hbase3:36809 {}] server.AbstractConnector(383): Stopped ServerConnector@31401045{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:27:48,697 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77ca6cd4 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77ca6cd4 2023-10-02T12:27:48,706 DEBUG [M:0;jenkins-hbase3:36809 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:27:48,706 INFO [M:0;jenkins-hbase3:36809 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:27:48,707 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249621173 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249621173,5,FailOnTimeoutGroup] 2023-10-02T12:27:48,707 DEBUG [M:0;jenkins-hbase3:36809 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:27:48,707 DEBUG [M:0;jenkins-hbase3:36809 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:27:48,707 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:27:48,707 INFO [M:0;jenkins-hbase3:36809 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:27:48,707 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249621174 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249621174,5,FailOnTimeoutGroup] 2023-10-02T12:27:48,707 DEBUG [M:0;jenkins-hbase3:36809 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:27:48,707 INFO [M:0;jenkins-hbase3:36809 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:27:48,707 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:27:48,708 ERROR [M:0;jenkins-hbase3:36809 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-10-02T12:27:48,708 INFO [M:0;jenkins-hbase3:36809 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:27:48,709 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:27:48,710 WARN [M:0;jenkins-hbase3:36809 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:27:48,710 INFO [M:0;jenkins-hbase3:36809 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:27:48,714 INFO [M:0;jenkins-hbase3:36809 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:27:48,718 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:27:48,736 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:27:48,736 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:27:48,736 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:27:48,736 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:27:48,774 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=285.08 KB heapSize=342.02 KB 2023-10-02T12:27:48,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:27:48,843 DEBUG [M:0;jenkins-hbase3:36809 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5335d1d025a6405b8646bbaeb54690c7 is 88, key is hbase:meta,,1/info:sn/1696249622022/Put/seqid=0 2023-10-02T12:27:48,908 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5335d1d025a6405b8646bbaeb54690c7 2023-10-02T12:27:48,993 DEBUG [M:0;jenkins-hbase3:36809 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb0a44eedddb478a9d992da67fef91fa is 1038, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1696249648059/Put/seqid=0 2023-10-02T12:27:49,466 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=284.49 KB at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb0a44eedddb478a9d992da67fef91fa 2023-10-02T12:27:49,516 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bb0a44eedddb478a9d992da67fef91fa 2023-10-02T12:27:49,545 DEBUG [M:0;jenkins-hbase3:36809 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e02135f0dad4a57b589ed0a74517523 is 82, key is jenkins-hbase3.apache.org,33719,1696249616681/rs:state/1696249621250/Put/seqid=0 2023-10-02T12:27:49,956 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=676 (bloomFilter=true), to=hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e02135f0dad4a57b589ed0a74517523 2023-10-02T12:27:49,968 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5335d1d025a6405b8646bbaeb54690c7 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5335d1d025a6405b8646bbaeb54690c7 2023-10-02T12:27:49,976 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5335d1d025a6405b8646bbaeb54690c7, entries=8, sequenceid=676, filesize=5.5 K 2023-10-02T12:27:49,978 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb0a44eedddb478a9d992da67fef91fa as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb0a44eedddb478a9d992da67fef91fa 2023-10-02T12:27:49,986 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bb0a44eedddb478a9d992da67fef91fa 2023-10-02T12:27:49,986 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb0a44eedddb478a9d992da67fef91fa, entries=75, sequenceid=676, filesize=23.2 K 2023-10-02T12:27:49,988 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2e02135f0dad4a57b589ed0a74517523 as hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2e02135f0dad4a57b589ed0a74517523 2023-10-02T12:27:50,007 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34623/user/jenkins/test-data/f89aa73d-1120-4255-2333-9fd03ac124fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2e02135f0dad4a57b589ed0a74517523, entries=1, sequenceid=676, filesize=5.1 K 2023-10-02T12:27:50,008 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(3022): Finished flush of dataSize ~285.08 KB/291925, heapSize ~341.72 KB/349920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1234ms, sequenceid=676, compaction requested=false 2023-10-02T12:27:50,022 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:27:50,022 DEBUG [M:0;jenkins-hbase3:36809 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:27:50,058 INFO [M:0;jenkins-hbase3:36809 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:27:50,058 INFO [M:0;jenkins-hbase3:36809 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:27:50,058 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:27:50,067 INFO [M:0;jenkins-hbase3:36809 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:36809 2023-10-02T12:27:50,292 INFO [M:0;jenkins-hbase3:36809 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,36809,1696249611352; zookeeper connection closed. 2023-10-02T12:27:50,295 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:27:50,321 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:27:50,321 WARN [512486321@qtp-1127294656-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36257 {}] http.HttpServer2$SelectChannelConnectorWithSafeStartup(546): HttpServer Acceptor: isRunning is false. Rechecking. 2023-10-02T12:27:50,321 WARN [512486321@qtp-1127294656-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36257 {}] http.HttpServer2$SelectChannelConnectorWithSafeStartup(555): HttpServer Acceptor: isRunning is false 2023-10-02T12:27:50,328 WARN [BP-432441259-172.31.12.81-1696249606345 heartbeating to localhost/127.0.0.1:34623 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:27:50,328 WARN [BP-432441259-172.31.12.81-1696249606345 heartbeating to localhost/127.0.0.1:34623 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-432441259-172.31.12.81-1696249606345 (Datanode Uuid 872d4aba-ee6e-4db2-8b67-32a5a5f4eaf8) service to localhost/127.0.0.1:34623 2023-10-02T12:27:50,330 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/dfs/data/data1/current/BP-432441259-172.31.12.81-1696249606345 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:27:50,331 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/aeadda05-9cdd-b6bc-eb48-351a1b5c3291/cluster_e2935788-2822-e4de-0fc0-ed4818728d06/dfs/data/data2/current/BP-432441259-172.31.12.81-1696249606345 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:27:50,375 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:27:50,577 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:27:50,577 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:27:50,602 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:27:50,726 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:27:50,838 ERROR [Thread[Thread-276,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:27:50,838 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:27:50,943 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:27:50,950 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:27:50,953 ERROR [Thread[Thread-286,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:27:50,960 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:27:50,963 ERROR [Thread[Thread-251,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:27:50,963 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped