2023-11-19 01:23:48,921 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a 2023-11-19 01:23:48,935 main DEBUG Took 0.011436 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-19 01:23:48,935 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-19 01:23:48,935 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-19 01:23:48,936 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-19 01:23:48,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,947 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-19 01:23:48,964 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,967 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,968 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,970 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,971 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,972 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,973 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,973 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,974 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,975 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:23:48,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,977 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-19 01:23:48,979 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:23:48,981 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-19 01:23:48,983 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-19 01:23:48,983 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-19 01:23:48,984 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-19 01:23:48,984 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-19 01:23:48,999 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-19 01:23:49,001 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-19 01:23:49,003 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-19 01:23:49,003 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-19 01:23:49,004 main DEBUG createAppenders(={Console}) 2023-11-19 01:23:49,004 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a initialized 2023-11-19 01:23:49,005 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a 2023-11-19 01:23:49,005 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a OK. 2023-11-19 01:23:49,005 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-19 01:23:49,006 main DEBUG OutputStream closed 2023-11-19 01:23:49,006 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-19 01:23:49,006 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-19 01:23:49,006 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@59fd97a8 OK 2023-11-19 01:23:49,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-19 01:23:49,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-19 01:23:49,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-19 01:23:49,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-19 01:23:49,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-19 01:23:49,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-19 01:23:49,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-19 01:23:49,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-19 01:23:49,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-19 01:23:49,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-19 01:23:49,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-19 01:23:49,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-19 01:23:49,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-19 01:23:49,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-19 01:23:49,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-19 01:23:49,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-19 01:23:49,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-19 01:23:49,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-19 01:23:49,063 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19 01:23:49,063 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5403f35f) with optional ClassLoader: null 2023-11-19 01:23:49,064 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-19 01:23:49,064 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@5403f35f] started OK. 2023-11-19T01:23:49,096 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupRepair timeout: 13 mins 2023-11-19 01:23:49,101 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-19 01:23:49,101 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19T01:23:49,441 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167 2023-11-19T01:23:49,446 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:23:49,448 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:23:49,450 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-19T01:23:49,453 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a, deleteOnExit=true 2023-11-19T01:23:49,453 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-19T01:23:49,454 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/test.cache.data in system properties and HBase conf 2023-11-19T01:23:49,454 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:23:49,455 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:23:49,455 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:23:49,455 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:23:49,456 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:23:49,571 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-19T01:23:50,176 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-19T01:23:50,181 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:23:50,181 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:23:50,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:23:50,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:23:50,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:23:50,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:23:50,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:23:50,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:23:50,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:23:50,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:23:50,186 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:23:50,186 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:23:50,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:23:50,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-19T01:23:50,673 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:23:50,678 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:23:51,010 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-19T01:23:51,186 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@52b4b4 via org.mortbay.log.Slf4jLog 2023-11-19T01:23:51,205 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:23:51,248 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:23:51,286 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir/Jetty_localhost_41707_hdfs____.fff2ph/webapp 2023-11-19T01:23:51,435 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41707 2023-11-19T01:23:51,457 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:23:51,458 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:23:52,007 WARN [Listener at localhost/45331 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:23:52,074 WARN [Listener at localhost/45331 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:23:52,093 WARN [Listener at localhost/45331 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:23:52,100 INFO [Listener at localhost/45331 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:23:52,105 INFO [Listener at localhost/45331 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir/Jetty_localhost_41591_datanode____.lrwt61/webapp 2023-11-19T01:23:52,206 INFO [Listener at localhost/45331 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41591 2023-11-19T01:23:52,506 WARN [Listener at localhost/35507 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:23:52,804 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:23:52,995 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb6ba2bb7f773c268: Processing first storage report for DS-c9c75027-187c-468b-be8d-a2e13e078574 from datanode 0182dfdc-c62a-4f7e-a76c-e909395d0474 2023-11-19T01:23:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb6ba2bb7f773c268: from storage DS-c9c75027-187c-468b-be8d-a2e13e078574 node DatanodeRegistration(127.0.0.1:37467, datanodeUuid=0182dfdc-c62a-4f7e-a76c-e909395d0474, infoPort=42111, infoSecurePort=0, ipcPort=35507, storageInfo=lv=-57;cid=testClusterID;nsid=441507692;c=1700357030750), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-11-19T01:23:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xb6ba2bb7f773c268: Processing first storage report for DS-9a231c28-90db-458b-a154-d350d5a3f080 from datanode 0182dfdc-c62a-4f7e-a76c-e909395d0474 2023-11-19T01:23:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xb6ba2bb7f773c268: from storage DS-9a231c28-90db-458b-a154-d350d5a3f080 node DatanodeRegistration(127.0.0.1:37467, datanodeUuid=0182dfdc-c62a-4f7e-a76c-e909395d0474, infoPort=42111, infoSecurePort=0, ipcPort=35507, storageInfo=lv=-57;cid=testClusterID;nsid=441507692;c=1700357030750), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:23:53,068 DEBUG [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167 2023-11-19T01:23:53,219 INFO [Listener at localhost/35507 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/zookeeper_0, clientPort=61923, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-19T01:23:53,241 INFO [Listener at localhost/35507 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=61923 2023-11-19T01:23:53,249 INFO [Listener at localhost/35507 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:53,251 INFO [Listener at localhost/35507 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:53,922 INFO [Listener at localhost/35507 {}] util.FSUtils(462): Created version file at hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 with version=8 2023-11-19T01:23:53,922 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/hbase-staging 2023-11-19T01:23:53,940 DEBUG [Listener at localhost/35507 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-19T01:23:53,940 DEBUG [Listener at localhost/35507 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-19T01:23:53,940 DEBUG [Listener at localhost/35507 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-19T01:23:53,940 DEBUG [Listener at localhost/35507 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-19T01:23:54,185 DEBUG [Listener at localhost/35507 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-19T01:23:55,015 INFO [Listener at localhost/35507 {}] client.ConnectionUtils(128): master/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:23:55,065 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:23:55,066 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:23:55,066 INFO [Listener at localhost/35507 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:23:55,066 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:23:55,066 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:23:55,240 INFO [Listener at localhost/35507 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:23:55,399 INFO [Listener at localhost/35507 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-19T01:23:55,427 DEBUG [Listener at localhost/35507 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-19T01:23:55,437 INFO [Listener at localhost/35507 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:23:55,509 DEBUG [Listener at localhost/35507 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 26878 (auto-detected) 2023-11-19T01:23:55,512 DEBUG [Listener at localhost/35507 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:55:ff:fe:37:d3:4d (auto-detected) 2023-11-19T01:23:55,567 INFO [Listener at localhost/35507 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:34089 2023-11-19T01:23:55,589 INFO [Listener at localhost/35507 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:55,591 INFO [Listener at localhost/35507 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:55,623 INFO [Listener at localhost/35507 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:34089 connecting to ZooKeeper ensemble=127.0.0.1:61923 2023-11-19T01:23:55,687 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:340890x0, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:23:55,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:34089-0x10123a159260000 connected 2023-11-19T01:23:55,742 DEBUG [Listener at localhost/35507 {}] zookeeper.ZKUtil(113): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:23:55,745 DEBUG [Listener at localhost/35507 {}] zookeeper.ZKUtil(113): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:23:55,749 DEBUG [Listener at localhost/35507 {}] zookeeper.ZKUtil(113): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:23:55,760 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34089 2023-11-19T01:23:55,762 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34089 2023-11-19T01:23:55,764 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34089 2023-11-19T01:23:55,766 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34089 2023-11-19T01:23:55,767 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34089 2023-11-19T01:23:55,822 INFO [Listener at localhost/35507 {}] log.Log(170): Logging initialized @7638ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-19T01:23:55,944 INFO [Listener at localhost/35507 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:23:55,945 INFO [Listener at localhost/35507 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:23:55,946 INFO [Listener at localhost/35507 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:23:55,948 INFO [Listener at localhost/35507 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-19T01:23:55,948 INFO [Listener at localhost/35507 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:23:55,948 INFO [Listener at localhost/35507 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:23:55,951 INFO [Listener at localhost/35507 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:23:56,083 INFO [Listener at localhost/35507 {}] http.HttpServer(1227): Jetty bound to port 38729 2023-11-19T01:23:56,085 INFO [Listener at localhost/35507 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:23:56,120 INFO [Listener at localhost/35507 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:23:56,124 INFO [Listener at localhost/35507 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@b7ce79c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:23:56,124 INFO [Listener at localhost/35507 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:23:56,124 INFO [Listener at localhost/35507 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2cfbfc75{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:23:56,195 INFO [Listener at localhost/35507 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:23:56,215 INFO [Listener at localhost/35507 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:23:56,215 INFO [Listener at localhost/35507 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:23:56,218 INFO [Listener at localhost/35507 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-19T01:23:56,229 INFO [Listener at localhost/35507 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:23:57,487 INFO [Listener at localhost/35507 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@661f9dfa{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:23:57,498 INFO [Listener at localhost/35507 {}] server.AbstractConnector(333): Started ServerConnector@76b72c0e{HTTP/1.1, (http/1.1)}{0.0.0.0:38729} 2023-11-19T01:23:57,499 INFO [Listener at localhost/35507 {}] server.Server(415): Started @9314ms 2023-11-19T01:23:57,502 INFO [Listener at localhost/35507 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132, hbase.cluster.distributed=false 2023-11-19T01:23:57,570 INFO [Listener at localhost/35507 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:23:57,571 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:23:57,571 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:23:57,571 INFO [Listener at localhost/35507 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:23:57,571 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:23:57,572 INFO [Listener at localhost/35507 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:23:57,583 INFO [Listener at localhost/35507 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:23:57,585 INFO [Listener at localhost/35507 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:23:57,586 INFO [Listener at localhost/35507 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:36757 2023-11-19T01:23:57,589 INFO [Listener at localhost/35507 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-19T01:23:57,597 DEBUG [Listener at localhost/35507 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-19T01:23:57,598 INFO [Listener at localhost/35507 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:57,601 INFO [Listener at localhost/35507 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:57,603 INFO [Listener at localhost/35507 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:36757 connecting to ZooKeeper ensemble=127.0.0.1:61923 2023-11-19T01:23:57,607 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:367570x0, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:23:57,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:36757-0x10123a159260001 connected 2023-11-19T01:23:57,608 DEBUG [Listener at localhost/35507 {}] zookeeper.ZKUtil(113): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:23:57,610 DEBUG [Listener at localhost/35507 {}] zookeeper.ZKUtil(113): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:23:57,611 DEBUG [Listener at localhost/35507 {}] zookeeper.ZKUtil(113): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:23:57,614 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36757 2023-11-19T01:23:57,617 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36757 2023-11-19T01:23:57,618 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36757 2023-11-19T01:23:57,621 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36757 2023-11-19T01:23:57,622 DEBUG [Listener at localhost/35507 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36757 2023-11-19T01:23:57,626 INFO [Listener at localhost/35507 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:23:57,627 INFO [Listener at localhost/35507 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:23:57,627 INFO [Listener at localhost/35507 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:23:57,628 INFO [Listener at localhost/35507 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-19T01:23:57,628 INFO [Listener at localhost/35507 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:23:57,628 INFO [Listener at localhost/35507 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:23:57,628 INFO [Listener at localhost/35507 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:23:57,630 INFO [Listener at localhost/35507 {}] http.HttpServer(1227): Jetty bound to port 43565 2023-11-19T01:23:57,631 INFO [Listener at localhost/35507 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:23:57,635 INFO [Listener at localhost/35507 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:23:57,635 INFO [Listener at localhost/35507 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@539fdaa1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:23:57,636 INFO [Listener at localhost/35507 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:23:57,636 INFO [Listener at localhost/35507 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@45f7ed9d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:23:57,649 INFO [Listener at localhost/35507 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:23:57,650 INFO [Listener at localhost/35507 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:23:57,650 INFO [Listener at localhost/35507 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:23:57,650 INFO [Listener at localhost/35507 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-19T01:23:57,655 INFO [Listener at localhost/35507 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:23:57,660 INFO [Listener at localhost/35507 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@7b99e2c7{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:23:57,661 INFO [Listener at localhost/35507 {}] server.AbstractConnector(333): Started ServerConnector@7cdacb0f{HTTP/1.1, (http/1.1)}{0.0.0.0:43565} 2023-11-19T01:23:57,661 INFO [Listener at localhost/35507 {}] server.Server(415): Started @9477ms 2023-11-19T01:23:57,667 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:23:57,676 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@601759f1{HTTP/1.1, (http/1.1)}{0.0.0.0:37691} 2023-11-19T01:23:57,676 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(415): Started @9492ms 2023-11-19T01:23:57,676 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:57,690 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:23:57,690 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:23:57,695 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:57,696 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase6:34089 2023-11-19T01:23:57,716 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:23:57,716 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:23:57,716 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:57,716 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:57,717 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:23:57,720 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:23:57,722 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase6.apache.org,34089,1700357034167 from backup master directory 2023-11-19T01:23:57,727 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:57,727 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:23:57,727 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:23:57,728 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:23:57,729 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:57,732 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-19T01:23:57,734 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-19T01:23:57,824 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/hbase.id with ID: 8e2eec2d-3cfb-4ce6-bb9a-8803be60536d 2023-11-19T01:23:57,867 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:23:57,886 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:57,886 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:57,930 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:23:57,932 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-19T01:23:57,952 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-19T01:23:57,952 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-19T01:23:57,954 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:23:57,965 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:23:57,967 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:23:58,447 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store 2023-11-19T01:23:58,472 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-19T01:23:58,472 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:23:58,473 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:23:58,473 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:23:58,473 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:23:58,473 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:23:58,473 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:23:58,473 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:23:58,473 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:23:58,475 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/.initializing 2023-11-19T01:23:58,475 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/WALs/jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:58,484 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:23:58,499 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C34089%2C1700357034167, suffix=, logDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/WALs/jenkins-hbase6.apache.org,34089,1700357034167, archiveDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/oldWALs, maxLogs=10 2023-11-19T01:23:58,536 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/WALs/jenkins-hbase6.apache.org,34089,1700357034167/jenkins-hbase6.apache.org%2C34089%2C1700357034167.1700357038506, exclude list is [], retry=0 2023-11-19T01:23:58,567 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:23:58,574 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:23:58,646 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/WALs/jenkins-hbase6.apache.org,34089,1700357034167/jenkins-hbase6.apache.org%2C34089%2C1700357034167.1700357038506 2023-11-19T01:23:58,647 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:23:58,648 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:23:58,648 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:23:58,653 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,654 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-19T01:23:58,773 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:23:58,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:23:58,777 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,780 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-19T01:23:58,780 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:23:58,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:23:58,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,784 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-19T01:23:58,784 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:23:58,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:23:58,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,788 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-19T01:23:58,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:23:58,789 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:23:58,793 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,794 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,812 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-19T01:23:58,815 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:23:58,819 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:23:58,820 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10459722720, jitterRate=-0.02586241066455841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-19T01:23:58,826 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:23:58,827 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-19T01:23:58,861 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c18323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:23:58,915 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-19T01:23:58,934 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-19T01:23:58,935 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-19T01:23:58,938 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-19T01:23:58,939 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-19T01:23:58,945 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 5 msec 2023-11-19T01:23:58,945 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-19T01:23:58,978 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-19T01:23:58,997 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-19T01:23:59,001 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-19T01:23:59,005 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-19T01:23:59,006 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-19T01:23:59,008 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-19T01:23:59,012 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-19T01:23:59,022 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-19T01:23:59,024 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-19T01:23:59,025 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-19T01:23:59,026 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-19T01:23:59,044 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-19T01:23:59,045 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-19T01:23:59,051 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:23:59,051 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:23:59,051 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,051 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,053 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase6.apache.org,34089,1700357034167, sessionid=0x10123a159260000, setting cluster-up flag (Was=false) 2023-11-19T01:23:59,063 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-19T01:23:59,075 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,075 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,081 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-19T01:23:59,083 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:59,088 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,088 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,095 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-19T01:23:59,096 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:59,101 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,101 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:23:59,108 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-19T01:23:59,109 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:23:59,175 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase6:36757 2023-11-19T01:23:59,177 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(997): ClusterId : 8e2eec2d-3cfb-4ce6-bb9a-8803be60536d 2023-11-19T01:23:59,179 INFO [RS:0;jenkins-hbase6:36757 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-19T01:23:59,181 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-19T01:23:59,187 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-19T01:23:59,187 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-19T01:23:59,191 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-19T01:23:59,191 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-19T01:23:59,195 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-19T01:23:59,196 DEBUG [RS:0;jenkins-hbase6:36757 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41780b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:23:59,198 DEBUG [RS:0;jenkins-hbase6:36757 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fea913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:23:59,205 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-19T01:23:59,205 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-19T01:23:59,205 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-19T01:23:59,208 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,34089,1700357034167 with isa=jenkins-hbase6.apache.org/172.31.2.20:36757, startcode=1700357037569 2023-11-19T01:23:59,228 DEBUG [RS:0;jenkins-hbase6:36757 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:23:59,264 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-19T01:23:59,278 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-19T01:23:59,286 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-19T01:23:59,295 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53883, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:23:59,299 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase6.apache.org,34089,1700357034167 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-19T01:23:59,305 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-19T01:23:59,305 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:23:59,305 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:23:59,306 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:23:59,306 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:23:59,306 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase6:0, corePoolSize=10, maxPoolSize=10 2023-11-19T01:23:59,306 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,306 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:23:59,307 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,311 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1700357069311 2023-11-19T01:23:59,314 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-19T01:23:59,321 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-19T01:23:59,325 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:23:59,326 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-19T01:23:59,336 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:23:59,336 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:23:59,337 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-19T01:23:59,337 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-19T01:23:59,338 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-19T01:23:59,339 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-19T01:23:59,339 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-19T01:23:59,342 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,347 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-19T01:23:59,350 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-19T01:23:59,351 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-19T01:23:59,356 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-19T01:23:59,356 WARN [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-19T01:23:59,368 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x31297faa to 127.0.0.1:61923 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:23:59,380 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f9d8df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:23:59,380 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:23:59,386 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-19T01:23:59,387 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-19T01:23:59,389 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357039389,5,FailOnTimeoutGroup] 2023-11-19T01:23:59,394 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357039389,5,FailOnTimeoutGroup] 2023-11-19T01:23:59,395 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,395 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-19T01:23:59,398 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,399 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,457 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,34089,1700357034167 with isa=jenkins-hbase6.apache.org/172.31.2.20:36757, startcode=1700357037569 2023-11-19T01:23:59,464 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34089 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:23:59,480 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:23:59,480 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:45331 2023-11-19T01:23:59,480 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38729 2023-11-19T01:23:59,492 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-19T01:23:59,493 DEBUG [RS:0;jenkins-hbase6:36757 {}] zookeeper.ZKUtil(111): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:23:59,493 WARN [RS:0;jenkins-hbase6:36757 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:23:59,493 INFO [RS:0;jenkins-hbase6:36757 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:23:59,494 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:23:59,497 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase6.apache.org,36757,1700357037569] 2023-11-19T01:23:59,516 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-19T01:23:59,536 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-19T01:23:59,561 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-19T01:23:59,566 INFO [RS:0;jenkins-hbase6:36757 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-19T01:23:59,566 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,567 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-19T01:23:59,577 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,577 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,577 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,578 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,579 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,579 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:23:59,579 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:23:59,579 DEBUG [RS:0;jenkins-hbase6:36757 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:23:59,580 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,580 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,580 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,580 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,580 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36757,1700357037569-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:23:59,600 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-19T01:23:59,603 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36757,1700357037569-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:23:59,620 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.Replication(204): jenkins-hbase6.apache.org,36757,1700357037569 started 2023-11-19T01:23:59,620 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase6.apache.org,36757,1700357037569, RpcServer on jenkins-hbase6.apache.org/172.31.2.20:36757, sessionid=0x10123a159260001 2023-11-19T01:23:59,621 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-19T01:23:59,621 DEBUG [RS:0;jenkins-hbase6:36757 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:23:59,621 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,36757,1700357037569' 2023-11-19T01:23:59,621 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-19T01:23:59,622 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-19T01:23:59,623 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-19T01:23:59,623 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-19T01:23:59,623 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,36757,1700357037569' 2023-11-19T01:23:59,623 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:23:59,624 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:23:59,624 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-19T01:23:59,624 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-19T01:23:59,625 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-19T01:23:59,625 DEBUG [RS:0;jenkins-hbase6:36757 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:23:59,625 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,36757,1700357037569' 2023-11-19T01:23:59,625 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-19T01:23:59,625 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-19T01:23:59,626 DEBUG [RS:0;jenkins-hbase6:36757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-19T01:23:59,626 INFO [RS:0;jenkins-hbase6:36757 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-19T01:23:59,626 INFO [RS:0;jenkins-hbase6:36757 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-19T01:23:59,737 INFO [RS:0;jenkins-hbase6:36757 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:23:59,740 INFO [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C36757%2C1700357037569, suffix=, logDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569, archiveDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/oldWALs, maxLogs=32 2023-11-19T01:23:59,753 DEBUG [RS:0;jenkins-hbase6:36757 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357039742, exclude list is [], retry=0 2023-11-19T01:23:59,759 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:23:59,800 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-19T01:23:59,800 INFO [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357039742 2023-11-19T01:23:59,800 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:23:59,800 DEBUG [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:24:00,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:00,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:24:00,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:24:00,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:00,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:24:00,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:24:00,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:24:00,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:00,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:24:00,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:24:00,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:24:00,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:00,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:24:00,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740 2023-11-19T01:24:00,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740 2023-11-19T01:24:00,233 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:24:00,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:24:00,238 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:00,239 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10921006080, jitterRate=0.017097949981689453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:24:00,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:24:00,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:24:00,241 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:24:00,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:24:00,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:24:00,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:24:00,243 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:24:00,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:24:00,247 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:24:00,247 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-19T01:24:00,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-19T01:24:00,269 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-19T01:24:00,273 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-19T01:24:00,425 DEBUG [jenkins-hbase6:34089 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-19T01:24:00,433 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:00,439 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,36757,1700357037569, state=OPENING 2023-11-19T01:24:00,449 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-19T01:24:00,451 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:24:00,451 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:24:00,452 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:24:00,452 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:24:00,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:00,758 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:00,761 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-19T01:24:00,764 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50594, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-19T01:24:00,776 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-19T01:24:00,776 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:24:00,777 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-19T01:24:00,780 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta, suffix=.meta, logDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569, archiveDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/oldWALs, maxLogs=32 2023-11-19T01:24:00,795 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357040782.meta, exclude list is [], retry=0 2023-11-19T01:24:00,799 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:24:00,803 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357040782.meta 2023-11-19T01:24:00,803 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:24:00,804 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:00,808 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:00,809 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-19T01:24:00,825 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-19T01:24:00,826 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-19T01:24:00,830 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-19T01:24:00,830 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:00,831 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-19T01:24:00,831 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-19T01:24:00,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:24:00,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:24:00,834 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:00,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:24:00,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:24:00,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:24:00,837 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:00,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:24:00,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:24:00,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:24:00,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:00,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:24:00,841 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740 2023-11-19T01:24:00,844 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740 2023-11-19T01:24:00,846 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:24:00,848 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:24:00,849 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9548865120, jitterRate=-0.11069263517856598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:24:00,850 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:24:00,862 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1700357040749 2023-11-19T01:24:00,881 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-19T01:24:00,881 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-19T01:24:00,882 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:00,884 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,36757,1700357037569, state=OPEN 2023-11-19T01:24:00,888 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:24:00,888 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:24:00,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:24:00,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:24:00,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-19T01:24:00,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,36757,1700357037569 in 434 msec 2023-11-19T01:24:00,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-19T01:24:00,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 637 msec 2023-11-19T01:24:00,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7270 sec 2023-11-19T01:24:00,907 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1700357040907, completionTime=-1 2023-11-19T01:24:00,907 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-19T01:24:00,907 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-19T01:24:00,981 DEBUG [hconnection-0x6ab7bfea-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:00,986 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50598, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:01,003 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-19T01:24:01,004 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1700357101003 2023-11-19T01:24:01,004 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1700357161004 2023-11-19T01:24:01,004 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 96 msec 2023-11-19T01:24:01,030 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34089,1700357034167-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:24:01,030 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34089,1700357034167-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:24:01,030 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34089,1700357034167-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:24:01,032 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase6:34089, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:24:01,032 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-19T01:24:01,039 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-19T01:24:01,052 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-19T01:24:01,054 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:24:01,063 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-19T01:24:01,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:01,069 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:01,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:01,487 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5859bbc8b279d13e33306d9bde605ca2, NAME => 'hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:01,899 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:01,899 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 5859bbc8b279d13e33306d9bde605ca2, disabling compactions & flushes 2023-11-19T01:24:01,899 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:01,900 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:01,900 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. after waiting 0 ms 2023-11-19T01:24:01,900 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:01,900 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:01,900 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5859bbc8b279d13e33306d9bde605ca2: 2023-11-19T01:24:01,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:01,917 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1700357041904"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357041904"}]},"ts":"1700357041904"} 2023-11-19T01:24:01,953 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:01,955 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:01,961 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357041955"}]},"ts":"1700357041955"} 2023-11-19T01:24:01,964 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-19T01:24:01,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5859bbc8b279d13e33306d9bde605ca2, ASSIGN}] 2023-11-19T01:24:01,972 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5859bbc8b279d13e33306d9bde605ca2, ASSIGN 2023-11-19T01:24:01,974 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=5859bbc8b279d13e33306d9bde605ca2, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:02,125 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5859bbc8b279d13e33306d9bde605ca2, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:02,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 5859bbc8b279d13e33306d9bde605ca2, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:02,283 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:02,289 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:02,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 5859bbc8b279d13e33306d9bde605ca2, NAME => 'hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:02,291 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:02,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:02,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,293 INFO [StoreOpener-5859bbc8b279d13e33306d9bde605ca2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,295 INFO [StoreOpener-5859bbc8b279d13e33306d9bde605ca2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5859bbc8b279d13e33306d9bde605ca2 columnFamilyName info 2023-11-19T01:24:02,295 DEBUG [StoreOpener-5859bbc8b279d13e33306d9bde605ca2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:02,296 INFO [StoreOpener-5859bbc8b279d13e33306d9bde605ca2-1 {}] regionserver.HStore(324): Store=5859bbc8b279d13e33306d9bde605ca2/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:02,298 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:02,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:02,306 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 5859bbc8b279d13e33306d9bde605ca2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10047191360, jitterRate=-0.0642823874950409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:24:02,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 5859bbc8b279d13e33306d9bde605ca2: 2023-11-19T01:24:02,309 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2., pid=6, masterSystemTime=1700357042283 2023-11-19T01:24:02,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:02,316 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:02,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5859bbc8b279d13e33306d9bde605ca2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:02,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-19T01:24:02,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 5859bbc8b279d13e33306d9bde605ca2, server=jenkins-hbase6.apache.org,36757,1700357037569 in 194 msec 2023-11-19T01:24:02,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-19T01:24:02,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=5859bbc8b279d13e33306d9bde605ca2, ASSIGN in 357 msec 2023-11-19T01:24:02,331 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:02,332 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357042331"}]},"ts":"1700357042331"} 2023-11-19T01:24:02,335 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-19T01:24:02,338 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:02,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2830 sec 2023-11-19T01:24:02,368 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-19T01:24:02,370 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:02,370 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36757-0x10123a159260001, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:24:02,370 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:24:02,420 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-19T01:24:02,436 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:02,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec 2023-11-19T01:24:02,454 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-19T01:24:02,465 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:02,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-11-19T01:24:02,483 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-19T01:24:02,494 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-19T01:24:02,494 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 4.765sec 2023-11-19T01:24:02,496 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-19T01:24:02,498 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-19T01:24:02,500 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-19T01:24:02,501 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-19T01:24:02,501 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-19T01:24:02,502 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34089,1700357034167-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:24:02,503 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,34089,1700357034167-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-19T01:24:02,525 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-19T01:24:02,527 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-19T01:24:02,585 DEBUG [Listener at localhost/35507 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5befaf2f to 127.0.0.1:61923 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:24:02,591 DEBUG [Listener at localhost/35507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8d326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:02,597 DEBUG [Listener at localhost/35507 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-19T01:24:02,597 DEBUG [Listener at localhost/35507 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-19T01:24:02,615 DEBUG [hconnection-0x3488e93c-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:02,628 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50606, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:02,639 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/test.cache.data in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:24:02,640 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:24:02,641 INFO [Listener at localhost/35507 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-19T01:24:04,133 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:24:04,512 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:24:04,519 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_33755_jobhistory____.mk5s4i/webapp 2023-11-19T01:24:04,698 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-19T01:24:04,698 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-19T01:24:04,698 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:24:04,701 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:24:04,790 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:04,911 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:05,161 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-19T01:24:05,195 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:33755 2023-11-19T01:24:05,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-19T01:24:05,393 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:05,399 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-19T01:24:05,399 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:05,401 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:05,401 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-19T01:24:05,401 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:24:05,401 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-19T01:24:05,401 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-19T01:24:05,402 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:05,526 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-19T01:24:05,528 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-19T01:24:05,632 WARN [Listener at jenkins-hbase6.apache.org/46679 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:24:05,717 WARN [Listener at jenkins-hbase6.apache.org/46679 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:24:05,718 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:24:05,724 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_34591_cluster____elkkyi/webapp 2023-11-19T01:24:05,793 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:24:05,794 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-19T01:24:05,794 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:24:05,794 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:24:05,803 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:06,038 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:06,181 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:06,191 INFO [Listener at jenkins-hbase6.apache.org/46679 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:34591 2023-11-19T01:24:06,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:06,649 WARN [Listener at jenkins-hbase6.apache.org/46287 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:24:06,672 WARN [Listener at jenkins-hbase6.apache.org/46287 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:24:06,673 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:24:06,678 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_46111_node____.26fzg5/webapp 2023-11-19T01:24:06,731 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:24:06,731 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:24:06,731 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:24:06,731 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:24:06,739 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:06,762 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:06,835 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:06,845 INFO [Listener at jenkins-hbase6.apache.org/46287 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:46111 2023-11-19T01:24:07,009 WARN [Listener at jenkins-hbase6.apache.org/34947 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:24:07,017 WARN [Listener at jenkins-hbase6.apache.org/34947 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:24:07,017 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:24:07,028 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_40743_node____laiq2e/webapp 2023-11-19T01:24:07,083 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:24:07,084 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:24:07,084 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:24:07,084 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:24:07,094 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:07,110 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:07,175 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:24:07,182 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:40743 2023-11-19T01:24:07,223 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-19T01:24:07,223 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:45331/backupUT 2023-11-19T01:24:07,235 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:07,239 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:38820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:07,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns1'} 2023-11-19T01:24:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-19T01:24:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:24:07,286 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:07,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 40 msec 2023-11-19T01:24:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:24:07,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns2'} 2023-11-19T01:24:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-19T01:24:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:24:07,381 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:07,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-11-19T01:24:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:24:07,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns3'} 2023-11-19T01:24:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-19T01:24:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:24:07,494 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:07,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-11-19T01:24:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:24:07,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns4'} 2023-11-19T01:24:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-19T01:24:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:24:07,603 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:07,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-11-19T01:24:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:24:07,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'test-1700357047223', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:24:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1700357047223 2023-11-19T01:24:07,717 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1700357047223 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:07,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "test-1700357047223" procId is: 13 2023-11-19T01:24:07,718 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:07,719 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1700357047223 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:24:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:24:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:24:08,156 INFO [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9c7b67ce00e8024f201b8cb1555d84b3, NAME => 'test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1700357047223', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:24:08,575 DEBUG [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(888): Instantiated test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:08,575 DEBUG [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1668): Closing 9c7b67ce00e8024f201b8cb1555d84b3, disabling compactions & flushes 2023-11-19T01:24:08,575 INFO [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1690): Closing region test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,575 DEBUG [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,575 DEBUG [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. after waiting 0 ms 2023-11-19T01:24:08,575 DEBUG [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,575 INFO [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1909): Closed test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,575 DEBUG [RegionOpenAndInit-test-1700357047223-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9c7b67ce00e8024f201b8cb1555d84b3: 2023-11-19T01:24:08,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1700357047223 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:08,577 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1700357048577"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357048577"}]},"ts":"1700357048577"} 2023-11-19T01:24:08,580 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:08,581 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1700357047223 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:08,581 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357047223","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357048581"}]},"ts":"1700357048581"} 2023-11-19T01:24:08,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357047223, state=ENABLING in hbase:meta 2023-11-19T01:24:08,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357047223, region=9c7b67ce00e8024f201b8cb1555d84b3, ASSIGN}] 2023-11-19T01:24:08,590 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357047223, region=9c7b67ce00e8024f201b8cb1555d84b3, ASSIGN 2023-11-19T01:24:08,592 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357047223, region=9c7b67ce00e8024f201b8cb1555d84b3, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:08,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=9c7b67ce00e8024f201b8cb1555d84b3, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:08,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 9c7b67ce00e8024f201b8cb1555d84b3, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:24:08,898 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:08,906 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,906 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 9c7b67ce00e8024f201b8cb1555d84b3, NAME => 'test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:08,906 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:08,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357047223 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:08,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,909 INFO [StoreOpener-9c7b67ce00e8024f201b8cb1555d84b3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,911 INFO [StoreOpener-9c7b67ce00e8024f201b8cb1555d84b3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7b67ce00e8024f201b8cb1555d84b3 columnFamilyName f 2023-11-19T01:24:08,911 DEBUG [StoreOpener-9c7b67ce00e8024f201b8cb1555d84b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:08,913 INFO [StoreOpener-9c7b67ce00e8024f201b8cb1555d84b3-1 {}] regionserver.HStore(324): Store=9c7b67ce00e8024f201b8cb1555d84b3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:08,914 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,915 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:08,928 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:08,929 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 9c7b67ce00e8024f201b8cb1555d84b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10654561760, jitterRate=-0.00771661102771759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:24:08,929 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 9c7b67ce00e8024f201b8cb1555d84b3: 2023-11-19T01:24:08,931 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3., pid=15, masterSystemTime=1700357048898 2023-11-19T01:24:08,933 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,933 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:08,934 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=9c7b67ce00e8024f201b8cb1555d84b3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:08,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-19T01:24:08,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 9c7b67ce00e8024f201b8cb1555d84b3, server=jenkins-hbase6.apache.org,36757,1700357037569 in 191 msec 2023-11-19T01:24:08,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-19T01:24:08,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1700357047223, region=9c7b67ce00e8024f201b8cb1555d84b3, ASSIGN in 351 msec 2023-11-19T01:24:08,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1700357047223 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:08,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357047223","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357048943"}]},"ts":"1700357048943"} 2023-11-19T01:24:08,945 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357047223, state=ENABLED in hbase:meta 2023-11-19T01:24:08,948 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1700357047223 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:08,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1700357047223 in 1.2350 sec 2023-11-19T01:24:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:24:09,832 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1700357047223, procId: 13 completed 2023-11-19T01:24:09,834 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6c96325f to 127.0.0.1:61923 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:24:09,840 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f67fb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:09,843 DEBUG [hconnection-0x790bfd6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:09,846 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50614, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:09,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36757 {}] regionserver.HRegion(8210): writing data to region test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:24:09,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns2:test-17003570472231', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:24:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-17003570472231 2023-11-19T01:24:09,966 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003570472231 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:09,966 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:09,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns2" qualifier: "test-17003570472231" procId is: 16 2023-11-19T01:24:09,967 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-17003570472231 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:24:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:24:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:24:10,390 INFO [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b3a631670a013c3c95d5f63d2f831b4e, NAME => 'ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17003570472231', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:24:10,805 DEBUG [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:10,806 DEBUG [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1668): Closing b3a631670a013c3c95d5f63d2f831b4e, disabling compactions & flushes 2023-11-19T01:24:10,806 INFO [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:10,806 DEBUG [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:10,806 DEBUG [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. after waiting 0 ms 2023-11-19T01:24:10,806 DEBUG [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:10,806 INFO [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:10,806 DEBUG [RegionOpenAndInit-ns2:test-17003570472231-pool-0 {}] regionserver.HRegion(1622): Region close journal for b3a631670a013c3c95d5f63d2f831b4e: 2023-11-19T01:24:10,807 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-17003570472231 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:10,808 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357050807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357050807"}]},"ts":"1700357050807"} 2023-11-19T01:24:10,810 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:10,811 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-17003570472231 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:10,811 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003570472231","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357050811"}]},"ts":"1700357050811"} 2023-11-19T01:24:10,813 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003570472231, state=ENABLING in hbase:meta 2023-11-19T01:24:10,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003570472231, region=b3a631670a013c3c95d5f63d2f831b4e, ASSIGN}] 2023-11-19T01:24:10,820 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003570472231, region=b3a631670a013c3c95d5f63d2f831b4e, ASSIGN 2023-11-19T01:24:10,821 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-17003570472231, region=b3a631670a013c3c95d5f63d2f831b4e, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:10,972 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=b3a631670a013c3c95d5f63d2f831b4e, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:10,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure b3a631670a013c3c95d5f63d2f831b4e, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:24:11,127 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:11,131 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:11,132 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => b3a631670a013c3c95d5f63d2f831b4e, NAME => 'ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:11,132 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:11,132 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003570472231 b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,132 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:11,132 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,132 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,134 INFO [StoreOpener-b3a631670a013c3c95d5f63d2f831b4e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,136 INFO [StoreOpener-b3a631670a013c3c95d5f63d2f831b4e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3a631670a013c3c95d5f63d2f831b4e columnFamilyName f 2023-11-19T01:24:11,136 DEBUG [StoreOpener-b3a631670a013c3c95d5f63d2f831b4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:11,137 INFO [StoreOpener-b3a631670a013c3c95d5f63d2f831b4e-1 {}] regionserver.HStore(324): Store=b3a631670a013c3c95d5f63d2f831b4e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:11,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,139 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,143 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:11,150 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:11,151 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened b3a631670a013c3c95d5f63d2f831b4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10371470080, jitterRate=-0.03408157825469971}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:24:11,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for b3a631670a013c3c95d5f63d2f831b4e: 2023-11-19T01:24:11,153 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e., pid=18, masterSystemTime=1700357051127 2023-11-19T01:24:11,156 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:11,156 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:11,156 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=b3a631670a013c3c95d5f63d2f831b4e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:11,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-19T01:24:11,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure b3a631670a013c3c95d5f63d2f831b4e, server=jenkins-hbase6.apache.org,36757,1700357037569 in 185 msec 2023-11-19T01:24:11,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-19T01:24:11,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003570472231, region=b3a631670a013c3c95d5f63d2f831b4e, ASSIGN in 344 msec 2023-11-19T01:24:11,166 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-17003570472231 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:11,166 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003570472231","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357051166"}]},"ts":"1700357051166"} 2023-11-19T01:24:11,168 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003570472231, state=ENABLED in hbase:meta 2023-11-19T01:24:11,172 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003570472231 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:11,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-17003570472231 in 1.2100 sec 2023-11-19T01:24:11,530 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17003570472231' 2023-11-19T01:24:11,530 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1700357047223' 2023-11-19T01:24:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:24:12,077 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-17003570472231, procId: 16 completed 2023-11-19T01:24:12,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36757 {}] regionserver.HRegion(8210): writing data to region ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:24:12,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns3:test-17003570472232', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:24:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-17003570472232 2023-11-19T01:24:12,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003570472232 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:12,167 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:12,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns3" qualifier: "test-17003570472232" procId is: 19 2023-11-19T01:24:12,168 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-17003570472232 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:24:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:24:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:24:12,588 INFO [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4350ab61ef4b67e9a8a8417c991edc1d, NAME => 'ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17003570472232', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:24:13,008 DEBUG [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:13,008 DEBUG [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1668): Closing 4350ab61ef4b67e9a8a8417c991edc1d, disabling compactions & flushes 2023-11-19T01:24:13,008 INFO [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,008 DEBUG [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,008 DEBUG [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. after waiting 0 ms 2023-11-19T01:24:13,008 DEBUG [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,008 INFO [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,008 DEBUG [RegionOpenAndInit-ns3:test-17003570472232-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4350ab61ef4b67e9a8a8417c991edc1d: 2023-11-19T01:24:13,009 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-17003570472232 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:13,010 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357053010"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357053010"}]},"ts":"1700357053010"} 2023-11-19T01:24:13,012 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:13,013 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-17003570472232 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:13,013 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003570472232","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357053013"}]},"ts":"1700357053013"} 2023-11-19T01:24:13,015 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003570472232, state=ENABLING in hbase:meta 2023-11-19T01:24:13,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003570472232, region=4350ab61ef4b67e9a8a8417c991edc1d, ASSIGN}] 2023-11-19T01:24:13,028 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003570472232, region=4350ab61ef4b67e9a8a8417c991edc1d, ASSIGN 2023-11-19T01:24:13,029 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-17003570472232, region=4350ab61ef4b67e9a8a8417c991edc1d, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:13,181 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=4350ab61ef4b67e9a8a8417c991edc1d, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:13,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 4350ab61ef4b67e9a8a8417c991edc1d, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:24:13,336 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:13,341 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,341 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 4350ab61ef4b67e9a8a8417c991edc1d, NAME => 'ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:13,341 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:13,341 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003570472232 4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,341 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:13,342 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,342 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,343 INFO [StoreOpener-4350ab61ef4b67e9a8a8417c991edc1d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,345 INFO [StoreOpener-4350ab61ef4b67e9a8a8417c991edc1d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4350ab61ef4b67e9a8a8417c991edc1d columnFamilyName f 2023-11-19T01:24:13,345 DEBUG [StoreOpener-4350ab61ef4b67e9a8a8417c991edc1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:13,346 INFO [StoreOpener-4350ab61ef4b67e9a8a8417c991edc1d-1 {}] regionserver.HStore(324): Store=4350ab61ef4b67e9a8a8417c991edc1d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:13,347 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns3/test-17003570472232/4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns3/test-17003570472232/4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,351 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:13,354 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns3/test-17003570472232/4350ab61ef4b67e9a8a8417c991edc1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:13,355 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 4350ab61ef4b67e9a8a8417c991edc1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9586290880, jitterRate=-0.10720708966255188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:24:13,356 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 4350ab61ef4b67e9a8a8417c991edc1d: 2023-11-19T01:24:13,357 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d., pid=21, masterSystemTime=1700357053336 2023-11-19T01:24:13,359 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,359 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:13,360 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=4350ab61ef4b67e9a8a8417c991edc1d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:13,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-19T01:24:13,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 4350ab61ef4b67e9a8a8417c991edc1d, server=jenkins-hbase6.apache.org,36757,1700357037569 in 179 msec 2023-11-19T01:24:13,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-19T01:24:13,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003570472232, region=4350ab61ef4b67e9a8a8417c991edc1d, ASSIGN in 340 msec 2023-11-19T01:24:13,368 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-17003570472232 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:13,368 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003570472232","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357053368"}]},"ts":"1700357053368"} 2023-11-19T01:24:13,370 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003570472232, state=ENABLED in hbase:meta 2023-11-19T01:24:13,373 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003570472232 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:13,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-17003570472232 in 1.2100 sec 2023-11-19T01:24:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:24:14,279 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-17003570472232, procId: 19 completed 2023-11-19T01:24:14,279 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-17003570472232 get assigned. Timeout = 60000ms 2023-11-19T01:24:14,281 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:24:14,286 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-17003570472232 assigned to meta. Checking AM states. 2023-11-19T01:24:14,287 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:24:14,287 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-17003570472232 assigned. 2023-11-19T01:24:14,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns4:test-17003570472233', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:24:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-17003570472233 2023-11-19T01:24:14,293 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003570472233 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:14,293 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:14,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns4" qualifier: "test-17003570472233" procId is: 22 2023-11-19T01:24:14,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-17003570472233 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:24:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:24:14,417 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:24:14,709 INFO [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cd1564ff9977411d21e3ea8e55f13843, NAME => 'ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17003570472233', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:14,722 DEBUG [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:14,722 DEBUG [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1668): Closing cd1564ff9977411d21e3ea8e55f13843, disabling compactions & flushes 2023-11-19T01:24:14,722 INFO [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:14,722 DEBUG [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:14,722 DEBUG [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. after waiting 0 ms 2023-11-19T01:24:14,722 DEBUG [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:14,722 INFO [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:14,722 DEBUG [RegionOpenAndInit-ns4:test-17003570472233-pool-0 {}] regionserver.HRegion(1622): Region close journal for cd1564ff9977411d21e3ea8e55f13843: 2023-11-19T01:24:14,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-17003570472233 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:14,724 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357054724"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357054724"}]},"ts":"1700357054724"} 2023-11-19T01:24:14,727 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:14,728 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-17003570472233 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:14,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003570472233","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357054728"}]},"ts":"1700357054728"} 2023-11-19T01:24:14,730 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003570472233, state=ENABLING in hbase:meta 2023-11-19T01:24:14,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003570472233, region=cd1564ff9977411d21e3ea8e55f13843, ASSIGN}] 2023-11-19T01:24:14,737 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003570472233, region=cd1564ff9977411d21e3ea8e55f13843, ASSIGN 2023-11-19T01:24:14,738 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-17003570472233, region=cd1564ff9977411d21e3ea8e55f13843, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:14,889 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=cd1564ff9977411d21e3ea8e55f13843, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:14,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure cd1564ff9977411d21e3ea8e55f13843, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:24:15,044 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:15,049 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:15,049 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => cd1564ff9977411d21e3ea8e55f13843, NAME => 'ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:15,049 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:15,049 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003570472233 cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,050 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:15,050 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,050 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,051 INFO [StoreOpener-cd1564ff9977411d21e3ea8e55f13843-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,053 INFO [StoreOpener-cd1564ff9977411d21e3ea8e55f13843-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd1564ff9977411d21e3ea8e55f13843 columnFamilyName f 2023-11-19T01:24:15,053 DEBUG [StoreOpener-cd1564ff9977411d21e3ea8e55f13843-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:15,054 INFO [StoreOpener-cd1564ff9977411d21e3ea8e55f13843-1 {}] regionserver.HStore(324): Store=cd1564ff9977411d21e3ea8e55f13843/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:15,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns4/test-17003570472233/cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,056 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns4/test-17003570472233/cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,059 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:15,061 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns4/test-17003570472233/cd1564ff9977411d21e3ea8e55f13843/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:15,062 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened cd1564ff9977411d21e3ea8e55f13843; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10015415840, jitterRate=-0.06724171340465546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:24:15,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for cd1564ff9977411d21e3ea8e55f13843: 2023-11-19T01:24:15,064 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843., pid=24, masterSystemTime=1700357055044 2023-11-19T01:24:15,066 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:15,066 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:15,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=cd1564ff9977411d21e3ea8e55f13843, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:15,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-19T01:24:15,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure cd1564ff9977411d21e3ea8e55f13843, server=jenkins-hbase6.apache.org,36757,1700357037569 in 176 msec 2023-11-19T01:24:15,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-19T01:24:15,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-17003570472233, region=cd1564ff9977411d21e3ea8e55f13843, ASSIGN in 335 msec 2023-11-19T01:24:15,074 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-17003570472233 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:15,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003570472233","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357055074"}]},"ts":"1700357055074"} 2023-11-19T01:24:15,076 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003570472233, state=ENABLED in hbase:meta 2023-11-19T01:24:15,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003570472233 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:15,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-17003570472233 in 790 msec 2023-11-19T01:24:15,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003570472233 2023-11-19T01:24:15,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003570472233 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:15,393 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1700357047223 2023-11-19T01:24:15,393 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1700357047223 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:15,394 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003570472232 2023-11-19T01:24:15,394 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003570472232 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:15,395 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003570472231 2023-11-19T01:24:15,395 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003570472231 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:24:15,400 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-17003570472233, procId: 22 completed 2023-11-19T01:24:15,401 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-17003570472233 get assigned. Timeout = 60000ms 2023-11-19T01:24:15,401 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:24:15,405 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-17003570472233 assigned to meta. Checking AM states. 2023-11-19T01:24:15,405 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:24:15,406 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-17003570472233 assigned. 2023-11-19T01:24:15,406 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6c96325f to 127.0.0.1:61923 2023-11-19T01:24:15,406 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:15,446 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.ResourceChecker(147): before: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=584, OpenFileDescriptor=689, MaxFileDescriptor=60000, SystemLoadAverage=204, ProcessCount=169, AvailableMemoryMB=3834 2023-11-19T01:24:15,446 WARN [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-11-19T01:24:15,450 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupRepair(57): Running stage 0 2023-11-19T01:24:15,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'backup'} 2023-11-19T01:24:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-19T01:24:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:24:15,477 DEBUG [Listener at localhost/35507-EventThread {}] zookeeper.ZKWatcher(604): master:34089-0x10123a159260000, quorum=127.0.0.1:61923, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:24:15,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 15 msec 2023-11-19T01:24:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:24:15,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:24:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-19T01:24:15,592 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:15,593 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:15,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-19T01:24:15,594 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:24:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:24:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:24:16,012 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:24:16,423 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:16,424 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing d634a0c0d7e9a4d882c1acc22f37a51c, disabling compactions & flushes 2023-11-19T01:24:16,424 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,424 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,424 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. after waiting 0 ms 2023-11-19T01:24:16,424 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,424 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,424 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:16,426 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:16,426 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357056426"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357056426"}]},"ts":"1700357056426"} 2023-11-19T01:24:16,429 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:16,430 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:16,430 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357056430"}]},"ts":"1700357056430"} 2023-11-19T01:24:16,432 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:24:16,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN}] 2023-11-19T01:24:16,439 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN 2023-11-19T01:24:16,439 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:16,591 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:16,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:24:16,745 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:16,750 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,750 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:16,751 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:16,751 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,751 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:16,751 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,751 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,753 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,755 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName meta 2023-11-19T01:24:16,756 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:16,756 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:16,756 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,758 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName session 2023-11-19T01:24:16,758 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:16,759 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:16,760 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,760 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,762 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:24:16,764 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:16,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:16,767 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened d634a0c0d7e9a4d882c1acc22f37a51c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9479959360, jitterRate=-0.1171099841594696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:24:16,768 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:16,769 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., pid=28, masterSystemTime=1700357056745 2023-11-19T01:24:16,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,771 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:16,771 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:16,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-19T01:24:16,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 180 msec 2023-11-19T01:24:16,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-19T01:24:16,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN in 338 msec 2023-11-19T01:24:16,778 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:16,778 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357056778"}]},"ts":"1700357056778"} 2023-11-19T01:24:16,780 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:24:16,783 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:16,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.1940 sec 2023-11-19T01:24:17,531 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17003570472233' 2023-11-19T01:24:17,532 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17003570472232' 2023-11-19T01:24:17,532 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:24:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:24:17,703 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-19T01:24:17,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:24:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-19T01:24:17,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:24:17,721 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:17,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-19T01:24:17,722 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:24:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:24:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:24:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:24:18,136 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => bf7d9872f21a90dc504641c20f40847a, NAME => 'backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132 2023-11-19T01:24:18,148 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:18,148 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing bf7d9872f21a90dc504641c20f40847a, disabling compactions & flushes 2023-11-19T01:24:18,148 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,148 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,148 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. after waiting 0 ms 2023-11-19T01:24:18,148 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,148 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,148 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for bf7d9872f21a90dc504641c20f40847a: 2023-11-19T01:24:18,150 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:24:18,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1700357058150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357058150"}]},"ts":"1700357058150"} 2023-11-19T01:24:18,153 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:18,154 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:24:18,154 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357058154"}]},"ts":"1700357058154"} 2023-11-19T01:24:18,156 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-19T01:24:18,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=bf7d9872f21a90dc504641c20f40847a, ASSIGN}] 2023-11-19T01:24:18,163 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=bf7d9872f21a90dc504641c20f40847a, ASSIGN 2023-11-19T01:24:18,165 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=bf7d9872f21a90dc504641c20f40847a, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,36757,1700357037569; forceNewPlan=false, retain=false 2023-11-19T01:24:18,316 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=bf7d9872f21a90dc504641c20f40847a, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:18,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure bf7d9872f21a90dc504641c20f40847a, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:24:18,471 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:18,476 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => bf7d9872f21a90dc504641c20f40847a, NAME => 'backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:18,476 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:18,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:18,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,479 INFO [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,480 INFO [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7d9872f21a90dc504641c20f40847a columnFamilyName meta 2023-11-19T01:24:18,480 DEBUG [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:18,481 INFO [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] regionserver.HStore(324): Store=bf7d9872f21a90dc504641c20f40847a/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:18,481 INFO [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,483 INFO [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7d9872f21a90dc504641c20f40847a columnFamilyName session 2023-11-19T01:24:18,483 DEBUG [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:18,483 INFO [StoreOpener-bf7d9872f21a90dc504641c20f40847a-1 {}] regionserver.HStore(324): Store=bf7d9872f21a90dc504641c20f40847a/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:18,485 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system_bulk/bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,485 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system_bulk/bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:24:18,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:18,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system_bulk/bf7d9872f21a90dc504641c20f40847a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:24:18,492 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened bf7d9872f21a90dc504641c20f40847a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9452746560, jitterRate=-0.11964437365531921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:24:18,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for bf7d9872f21a90dc504641c20f40847a: 2023-11-19T01:24:18,494 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a., pid=31, masterSystemTime=1700357058471 2023-11-19T01:24:18,496 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,496 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:18,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=bf7d9872f21a90dc504641c20f40847a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:18,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-19T01:24:18,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure bf7d9872f21a90dc504641c20f40847a, server=jenkins-hbase6.apache.org,36757,1700357037569 in 181 msec 2023-11-19T01:24:18,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-19T01:24:18,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=bf7d9872f21a90dc504641c20f40847a, ASSIGN in 340 msec 2023-11-19T01:24:18,504 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:24:18,504 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357058504"}]},"ts":"1700357058504"} 2023-11-19T01:24:18,506 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-19T01:24:18,509 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:24:18,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 792 msec 2023-11-19T01:24:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:24:18,830 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-19T01:24:18,880 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b378a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:18,881 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:18,883 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56872, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:18,888 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:18,889 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35786, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:18,957 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:24:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36757 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-19T01:24:19,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357059004 (current time:1700357059004). 2023-11-19T01:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:24:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2b6846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:19,023 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35794, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-19T01:24:19,049 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:24:19,055 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:19,071 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:19,074 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:19,088 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:24:19,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c}] 2023-11-19T01:24:19,100 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:24:19,260 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:19,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36757 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-19T01:24:19,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:19,264 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-19T01:24:19,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/ddf84a84c0ff450a96ad47de731903b5 is 49, key is activesession:/session:c/1700357058983/Put/seqid=0 2023-11-19T01:24:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:24:19,357 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:19,432 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/ddf84a84c0ff450a96ad47de731903b5 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:19,442 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5, entries=1, sequenceid=5, filesize=4.9 K 2023-11-19T01:24:19,447 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 183ms, sequenceid=5, compaction requested=false 2023-11-19T01:24:19,449 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:19,449 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. for snapshot_backup_system completed. 2023-11-19T01:24:19,450 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:24:19,457 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:24:19,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:24:19,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5] hfiles 2023-11-19T01:24:19,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 for snapshot=snapshot_backup_system 2023-11-19T01:24:19,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:19,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-19T01:24:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-19T01:24:19,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:19,485 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:19,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-19T01:24:19,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c in 395 msec 2023-11-19T01:24:19,488 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:24:19,490 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:19,493 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:19,493 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:24:19,496 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:24:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:24:19,919 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:19,932 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:19,933 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:24:19,935 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:19,935 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-19T01:24:19,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 897 msec 2023-11-19T01:24:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:24:20,160 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(115): Backup backup_1700357058926 started at 1700357060160. 2023-11-19T01:24:20,182 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(119): Backup session backup_1700357058926 has been started. Backup session finished. Status: FAILURE 2023-11-19T01:24:20,183 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:20,183 ERROR [Listener at jenkins-hbase6.apache.org/34947 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] ... 32 more 2023-11-19T01:24:20,192 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0bbfa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:20,198 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17894b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:20,199 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:20,200 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56878, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:20,205 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:20,206 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35798, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1700357058926,startts=1700357060160,failedts=1700357060225,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1700357058926 2023-11-19T01:24:20,225 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:24:20,227 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:24:20,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:24:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:24:20,258 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357060258"}]},"ts":"1700357060258"} 2023-11-19T01:24:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:24:20,259 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:24:20,262 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:24:20,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN}] 2023-11-19T01:24:20,265 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN 2023-11-19T01:24:20,266 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:20,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:24:20,428 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:20,429 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(119): Close d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:20,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1668): Closing d634a0c0d7e9a4d882c1acc22f37a51c, disabling compactions & flushes 2023-11-19T01:24:20,430 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1690): Closing region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:20,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:20,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. after waiting 0 ms 2023-11-19T01:24:20,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:20,430 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=214 B heapSize=800 B 2023-11-19T01:24:20,439 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/a6cb8439b0484a70834ea106a0a00289 is 218, key is session:backup_1700357058926/session:context/1700357060181/Put/seqid=0 2023-11-19T01:24:20,554 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:24:20,848 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/a6cb8439b0484a70834ea106a0a00289 2023-11-19T01:24:20,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/a6cb8439b0484a70834ea106a0a00289 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/a6cb8439b0484a70834ea106a0a00289 2023-11-19T01:24:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:24:20,867 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/a6cb8439b0484a70834ea106a0a00289, entries=1, sequenceid=9, filesize=5.2 K 2023-11-19T01:24:20,868 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 438ms, sequenceid=9, compaction requested=false 2023-11-19T01:24:20,876 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-11-19T01:24:20,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:20,879 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1909): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:20,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1622): Region close journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:20,881 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(163): Closed d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:20,882 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSED 2023-11-19T01:24:20,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-11-19T01:24:20,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 615 msec 2023-11-19T01:24:20,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-19T01:24:20,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN in 621 msec 2023-11-19T01:24:20,888 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357060888"}]},"ts":"1700357060888"} 2023-11-19T01:24:20,889 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:24:20,891 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:24:20,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; DisableTableProcedure table=backup:system in 644 msec 2023-11-19T01:24:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:24:21,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357061376 (current time:1700357061376). 2023-11-19T01:24:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357061375 VERSION not specified, setting to 2 2023-11-19T01:24:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a569bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:21,387 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37838, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-11-19T01:24:21,391 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:24:21,392 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:21,395 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357061375 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:21,396 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:24:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:24:21,808 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:24:21,810 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:24:21,810 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:24:21,810 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:21,811 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(327): No files under family: meta 2023-11-19T01:24:21,811 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:21,813 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/a6cb8439b0484a70834ea106a0a00289, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5] hfiles 2023-11-19T01:24:21,813 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/a6cb8439b0484a70834ea106a0a00289 2023-11-19T01:24:21,813 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:24:22,226 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:24:22,227 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:22,228 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:22,228 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357061375 2023-11-19T01:24:22,229 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357061375 2023-11-19T01:24:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:24:22,644 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:22,653 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:22,654 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357061375 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357061375 2023-11-19T01:24:22,656 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:22,656 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-11-19T01:24:22,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357061375 table=backup:system type=FLUSH ttl=0 } in 1.2680 sec 2023-11-19T01:24:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:24:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357059004 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:24:23,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:24:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:24:23,533 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-19T01:24:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:24:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:24:23,946 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-11-19T01:24:23,946 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-19T01:24:23,948 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:23,956 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357059004 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:23,957 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system 2023-11-19T01:24:23,961 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:24:23,961 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:23,971 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:23,982 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/a6cb8439b0484a70834ea106a0a00289 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/a6cb8439b0484a70834ea106a0a00289 2023-11-19T01:24:23,983 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:23,983 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357059004 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:23,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357063985"}]},"ts":"9223372036854775807"} 2023-11-19T01:24:24,041 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:24:24,041 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:24,041 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357063986"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357063986"}]},"ts":"1700357063986"} 2023-11-19T01:24:24,043 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:24,043 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:24:24,043 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:24,044 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:24:24,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357059004 type: FLUSH version: 2 ttl: 0 ) in 531 msec 2023-11-19T01:24:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:24:24,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357061375" 2023-11-19T01:24:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357061375 2023-11-19T01:24:24,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:24:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:24:24,162 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:24:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:24:24,163 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357064163"}]},"ts":"1700357064163"} 2023-11-19T01:24:24,164 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:24:24,167 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:24:24,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN}] 2023-11-19T01:24:24,169 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN 2023-11-19T01:24:24,170 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:24:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:24:24,321 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:24,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:24:24,475 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:24,479 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:24,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:24,480 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:24,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:24,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,482 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,483 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName meta 2023-11-19T01:24:24,483 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:24,484 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:24,484 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,485 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName session 2023-11-19T01:24:24,485 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:24,493 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:24,494 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:24,495 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,496 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:24:24,500 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:24,501 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened d634a0c0d7e9a4d882c1acc22f37a51c; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10427316480, jitterRate=-0.02888047695159912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:24:24,501 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:24,502 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., pid=41, masterSystemTime=1700357064475 2023-11-19T01:24:24,504 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:24,504 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:24,505 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPEN, openSeqNum=13, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:24,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-11-19T01:24:24,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 183 msec 2023-11-19T01:24:24,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-19T01:24:24,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN in 341 msec 2023-11-19T01:24:24,511 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357064511"}]},"ts":"1700357064511"} 2023-11-19T01:24:24,512 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:24:24,514 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:24:24,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; EnableTableProcedure table=backup:system in 362 msec 2023-11-19T01:24:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:24:24,767 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:24:24,767 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:24:24,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:24:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:24:24,777 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:24:24,778 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-19T01:24:24,780 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1700357058926 2023-11-19T01:24:24,795 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:24,795 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:24,815 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupRepair(57): Running stage 1 2023-11-19T01:24:24,834 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb0b8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:24,835 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:24,837 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35346, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:24,840 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:24,842 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:24,881 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:24:24,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357064886 (current time:1700357064886). 2023-11-19T01:24:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:24:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc180cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:24,896 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-11-19T01:24:24,901 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:24:24,902 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:24,905 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:24,907 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:24:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:24:25,316 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:24:25,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c}] 2023-11-19T01:24:25,318 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:25,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:24:25,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:25,393 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-19T01:24:25,393 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:25,469 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:25,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36757 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-11-19T01:24:25,470 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:25,472 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-11-19T01:24:25,487 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/7670c7518979419f8bdd8eeb610c071b is 284, key is session:backup_1700357058926/session:context/1700357064778/Put/seqid=0 2023-11-19T01:24:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:24:25,897 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:25,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/7670c7518979419f8bdd8eeb610c071b as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:25,912 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b, entries=2, sequenceid=18, filesize=5.2 K 2023-11-19T01:24:25,914 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 442ms, sequenceid=18, compaction requested=false 2023-11-19T01:24:25,914 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. for snapshot_backup_system completed. 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b] hfiles 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 for snapshot=snapshot_backup_system 2023-11-19T01:24:25,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b for snapshot=snapshot_backup_system 2023-11-19T01:24:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:24:26,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:26,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-11-19T01:24:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster(4026): Remote procedure done, pid=43 2023-11-19T01:24:26,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:26,327 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:26,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-11-19T01:24:26,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c in 1.0120 sec 2023-11-19T01:24:26,330 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:24:26,331 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:26,332 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:26,332 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:24:26,333 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:24:26,353 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:26,361 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:26,362 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:24:26,364 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:26,364 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-11-19T01:24:26,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4670 sec 2023-11-19T01:24:26,383 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-19T01:24:27,008 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(115): Backup backup_1700357064867 started at 1700357067008. 2023-11-19T01:24:27,010 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(119): Backup session backup_1700357064867 has been started. Backup session finished. Status: FAILURE 2023-11-19T01:24:27,014 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:27,014 ERROR [Listener at jenkins-hbase6.apache.org/34947 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] ... 32 more 2023-11-19T01:24:27,023 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378a8326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:27,028 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7aa3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:27,029 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:27,031 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:27,035 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:27,037 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37872, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1700357064867,startts=1700357067008,failedts=1700357067053,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1700357064867 2023-11-19T01:24:27,053 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:24:27,054 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:24:27,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:24:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:24:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:24:27,065 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357067065"}]},"ts":"1700357067065"} 2023-11-19T01:24:27,067 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:24:27,071 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:24:27,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN}] 2023-11-19T01:24:27,073 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN 2023-11-19T01:24:27,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:27,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:24:27,227 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:27,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:27,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing d634a0c0d7e9a4d882c1acc22f37a51c, disabling compactions & flushes 2023-11-19T01:24:27,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:27,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:27,228 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. after waiting 0 ms 2023-11-19T01:24:27,228 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:27,228 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=289 B heapSize=952 B 2023-11-19T01:24:27,242 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/c10b43ea01aa44969ac56aa846745b95 is 79, key is startcode:hdfs://localhost:45331/backupUT/meta:startcode/1700357067012/Put/seqid=0 2023-11-19T01:24:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:24:27,657 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=75 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/c10b43ea01aa44969ac56aa846745b95 2023-11-19T01:24:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:24:27,669 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/c1dccfce61fa4d5d820e8f8b7b2c3047 is 218, key is session:backup_1700357064867/session:context/1700357067009/Put/seqid=0 2023-11-19T01:24:27,676 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/c1dccfce61fa4d5d820e8f8b7b2c3047 2023-11-19T01:24:27,683 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/c10b43ea01aa44969ac56aa846745b95 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/c10b43ea01aa44969ac56aa846745b95 2023-11-19T01:24:27,689 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/c10b43ea01aa44969ac56aa846745b95, entries=1, sequenceid=23, filesize=5.1 K 2023-11-19T01:24:27,690 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/c1dccfce61fa4d5d820e8f8b7b2c3047 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/c1dccfce61fa4d5d820e8f8b7b2c3047 2023-11-19T01:24:27,698 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/c1dccfce61fa4d5d820e8f8b7b2c3047, entries=1, sequenceid=23, filesize=5.2 K 2023-11-19T01:24:27,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(3022): Finished flush of dataSize ~289 B/289, heapSize ~920 B/920, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 471ms, sequenceid=23, compaction requested=true 2023-11-19T01:24:27,712 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=12 2023-11-19T01:24:27,712 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:27,713 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:27,713 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:27,715 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:27,715 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSED 2023-11-19T01:24:27,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-19T01:24:27,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 642 msec 2023-11-19T01:24:27,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-11-19T01:24:27,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN in 648 msec 2023-11-19T01:24:27,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357067721"}]},"ts":"1700357067721"} 2023-11-19T01:24:27,723 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:24:27,725 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:24:27,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=backup:system in 665 msec 2023-11-19T01:24:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:24:28,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357068179 (current time:1700357068179). 2023-11-19T01:24:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357068179 VERSION not specified, setting to 2 2023-11-19T01:24:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84f9450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:28,189 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-11-19T01:24:28,193 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:24:28,194 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:28,197 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357068179 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:28,200 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:24:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:24:28,614 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:24:28,616 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:24:28,616 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:24:28,616 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:28,617 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/c10b43ea01aa44969ac56aa846745b95] hfiles 2023-11-19T01:24:28,617 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/c10b43ea01aa44969ac56aa846745b95 2023-11-19T01:24:28,617 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:28,619 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/c1dccfce61fa4d5d820e8f8b7b2c3047, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5] hfiles 2023-11-19T01:24:28,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:28,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/c1dccfce61fa4d5d820e8f8b7b2c3047 2023-11-19T01:24:28,620 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:24:29,029 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:24:29,030 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:29,031 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:29,031 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357068179 2023-11-19T01:24:29,032 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357068179 2023-11-19T01:24:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:24:29,443 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:29,452 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:29,453 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357068179 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357068179 2023-11-19T01:24:29,454 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:29,454 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-11-19T01:24:29,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357068179 table=backup:system type=FLUSH ttl=0 } in 1.2640 sec 2023-11-19T01:24:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:24:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357064886 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:24:30,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:24:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:24:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:24:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:24:30,730 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-19T01:24:30,730 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-19T01:24:30,731 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:30,734 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357064886 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:30,734 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system 2023-11-19T01:24:30,737 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:24:30,737 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:30,745 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/c10b43ea01aa44969ac56aa846745b95 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/c10b43ea01aa44969ac56aa846745b95 2023-11-19T01:24:30,745 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:30,749 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/c1dccfce61fa4d5d820e8f8b7b2c3047 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/c1dccfce61fa4d5d820e8f8b7b2c3047 2023-11-19T01:24:30,749 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:30,749 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357064886 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:30,751 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357070750"}]},"ts":"9223372036854775807"} 2023-11-19T01:24:30,753 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:24:30,753 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:30,753 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357070751"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357070751"}]},"ts":"1700357070751"} 2023-11-19T01:24:30,754 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:30,755 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:24:30,755 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:30,755 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:24:30,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357064886 type: FLUSH version: 2 ttl: 0 ) in 447 msec 2023-11-19T01:24:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:24:30,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357068179" 2023-11-19T01:24:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357068179 2023-11-19T01:24:30,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:24:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:24:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:24:30,925 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:24:30,925 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357070925"}]},"ts":"1700357070925"} 2023-11-19T01:24:30,927 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:24:30,930 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:24:30,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN}] 2023-11-19T01:24:30,932 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN 2023-11-19T01:24:30,933 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:24:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:24:31,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:31,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:31,201 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:24:31,204 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:24:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:24:31,238 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:31,241 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(130): Open backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:31,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7254): Opening region: {ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:31,242 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:31,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(888): Instantiated backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:31,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7296): checking encryption for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7299): checking classloading for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,243 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,244 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName meta 2023-11-19T01:24:31,244 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:31,245 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:31,245 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,246 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName session 2023-11-19T01:24:31,246 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:31,254 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:31,259 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:31,259 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:31,260 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,261 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,263 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:24:31,264 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1076): writing seq id for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:31,265 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): Opened d634a0c0d7e9a4d882c1acc22f37a51c; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11634278880, jitterRate=0.08352665603160858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:24:31,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(992): Region open journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:31,266 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., pid=51, masterSystemTime=1700357071237 2023-11-19T01:24:31,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:31,268 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(158): Opened backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:31,268 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPEN, openSeqNum=27, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:31,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-19T01:24:31,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 184 msec 2023-11-19T01:24:31,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=50, resume processing ppid=49 2023-11-19T01:24:31,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, ppid=49, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN in 341 msec 2023-11-19T01:24:31,273 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357071273"}]},"ts":"1700357071273"} 2023-11-19T01:24:31,274 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:24:31,276 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:24:31,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; EnableTableProcedure table=backup:system in 356 msec 2023-11-19T01:24:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:24:31,528 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:24:31,528 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:24:31,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:24:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:24:31,537 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:24:31,537 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-19T01:24:31,539 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1700357064867 2023-11-19T01:24:31,544 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:31,544 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:31,575 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupRepair(57): Running stage 2 2023-11-19T01:24:31,601 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c7a9a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:31,602 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:31,603 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47340, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:31,607 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:31,613 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55866, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:31,657 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:24:31,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357071661 (current time:1700357071661). 2023-11-19T01:24:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:24:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b565f04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:31,674 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55876, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-19T01:24:31,678 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:24:31,679 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:31,681 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:31,684 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:24:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:24:32,093 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:24:32,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c}] 2023-11-19T01:24:32,095 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:32,246 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:32,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36757 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-11-19T01:24:32,267 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:32,269 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-11-19T01:24:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:24:32,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/7796765c6c0d4267b772f182b8abd1e8 is 284, key is session:backup_1700357064867/session:context/1700357071538/Put/seqid=0 2023-11-19T01:24:32,691 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/7796765c6c0d4267b772f182b8abd1e8 2023-11-19T01:24:32,699 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/7796765c6c0d4267b772f182b8abd1e8 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8 2023-11-19T01:24:32,705 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8, entries=2, sequenceid=32, filesize=5.2 K 2023-11-19T01:24:32,706 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 437ms, sequenceid=32, compaction requested=true 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. for snapshot_backup_system completed. 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8] hfiles 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 for snapshot=snapshot_backup_system 2023-11-19T01:24:32,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b for snapshot=snapshot_backup_system 2023-11-19T01:24:32,707 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8 for snapshot=snapshot_backup_system 2023-11-19T01:24:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:24:33,112 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:33,112 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-11-19T01:24:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster(4026): Remote procedure done, pid=53 2023-11-19T01:24:33,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:33,114 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:33,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-11-19T01:24:33,116 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:24:33,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c in 1.0220 sec 2023-11-19T01:24:33,117 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:33,117 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:33,118 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:24:33,118 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:24:33,216 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:33,529 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:33,537 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:33,537 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:24:33,539 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:33,539 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-19T01:24:33,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8630 sec 2023-11-19T01:24:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:24:33,785 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(115): Backup backup_1700357071642 started at 1700357073785. 2023-11-19T01:24:33,787 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(119): Backup session backup_1700357071642 has been started. 2023-11-19T01:24:33,790 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-11-19T01:24:33,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:24:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:24:33,806 INFO [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:24:33,806 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:24:33,808 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:24:33,808 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:24:33,809 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:24:33,814 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,814 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:24:33,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:24:33,815 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:24:33,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:24:33,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:24:33,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:24:33,816 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:24:33,817 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:24:33,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:24:33,818 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:24:33,818 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:24:33,819 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:24:33,819 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:24:33,819 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,36757,1700357037569' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:24:33,826 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:33,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:24:33,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:24:33,827 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:24:33,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:24:33,828 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:24:33,828 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:33,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:24:33,830 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,36757,1700357037569' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:24:33,830 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:24:33,830 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1c16e99d[Count = 0] remaining members to acquire global barrier 2023-11-19T01:24:33,830 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:33,834 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:33,834 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:33,834 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:24:33,834 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:24:33,835 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:24:33,835 DEBUG [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,835 INFO [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357039742 highest: 1700357039742 on jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,835 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta:.meta(num 1700357040782) roll requested 2023-11-19T01:24:33,852 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357073836.meta, exclude list is [], retry=0 2023-11-19T01:24:33,856 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:24:33,859 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357040782.meta with entries=56, filesize=15.96 KB; new WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357073836.meta 2023-11-19T01:24:33,860 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:24:33,860 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357040782.meta is not closed yet, will try archiving it next time 2023-11-19T01:24:33,860 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C36757%2C1700357037569:(num 1700357039742) roll requested 2023-11-19T01:24:33,872 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357073860, exclude list is [], retry=0 2023-11-19T01:24:33,875 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:24:33,878 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357039742 with entries=40, filesize=9.34 KB; new WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357073860 2023-11-19T01:24:33,878 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:24:33,878 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357039742 is not closed yet, will try archiving it next time 2023-11-19T01:24:33,896 DEBUG [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-11-19T01:24:33,896 INFO [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357073860 on jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,898 DEBUG [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:33,899 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47356, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:24:33,915 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:24:33,915 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,36757,1700357037569' in zk 2023-11-19T01:24:33,918 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:24:33,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,918 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:24:33,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:24:33,918 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:24:33,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:24:33,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:24:33,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:24:33,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:33,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:24:33,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:33,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,36757,1700357037569': 2023-11-19T01:24:33,922 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,36757,1700357037569' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:24:33,922 INFO [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:24:33,922 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:24:33,922 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:24:33,922 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:24:33,922 INFO [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:24:33,925 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:24:33,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:24:33,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:24:33,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:24:33,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:24:33,926 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:24:33,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:24:33,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:24:33,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:24:33,927 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:33,927 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:24:33,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:33,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:24:33,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:33,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:33,939 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:24:33,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:24:33,939 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:24:33,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:24:33,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:24:33,939 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:24:33,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:24:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done Backup session finished. Status: FAILURE 2023-11-19T01:24:34,049 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:34,049 ERROR [Listener at jenkins-hbase6.apache.org/34947 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] ... 32 more 2023-11-19T01:24:34,058 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4a76a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:34,064 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e63ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:34,065 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:34,066 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:34,070 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:34,071 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55886, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1700357071642,startts=1700357073785,failedts=1700357074089,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1700357071642 2023-11-19T01:24:34,089 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:24:34,090 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:24:34,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:24:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:24:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:24:34,099 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357074099"}]},"ts":"1700357074099"} 2023-11-19T01:24:34,100 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:24:34,102 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:24:34,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN}] 2023-11-19T01:24:34,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN 2023-11-19T01:24:34,104 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:34,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:24:34,257 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:34,258 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:34,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing d634a0c0d7e9a4d882c1acc22f37a51c, disabling compactions & flushes 2023-11-19T01:24:34,258 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:34,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:34,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. after waiting 0 ms 2023-11-19T01:24:34,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:34,258 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-11-19T01:24:34,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/a11bba30e02649f4a831043ab98ed269 is 116, key is rslogts:hdfs://localhost:45331/backupUT\x00jenkins-hbase6.apache.org:36757/meta:rs-log-ts/1700357073914/Put/seqid=0 2023-11-19T01:24:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:24:34,679 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/a11bba30e02649f4a831043ab98ed269 2023-11-19T01:24:34,690 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/fe42e354ed8a48c49c27d01261963df4 is 218, key is session:backup_1700357071642/session:context/1700357073786/Put/seqid=0 2023-11-19T01:24:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:24:35,095 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/fe42e354ed8a48c49c27d01261963df4 2023-11-19T01:24:35,103 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/a11bba30e02649f4a831043ab98ed269 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/a11bba30e02649f4a831043ab98ed269 2023-11-19T01:24:35,110 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/a11bba30e02649f4a831043ab98ed269, entries=2, sequenceid=38, filesize=5.3 K 2023-11-19T01:24:35,112 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/fe42e354ed8a48c49c27d01261963df4 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe42e354ed8a48c49c27d01261963df4 2023-11-19T01:24:35,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe42e354ed8a48c49c27d01261963df4, entries=1, sequenceid=38, filesize=5.2 K 2023-11-19T01:24:35,120 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 862ms, sequenceid=38, compaction requested=true 2023-11-19T01:24:35,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/recovered.edits/41.seqid, newMaxSeqId=41, maxSeqId=26 2023-11-19T01:24:35,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:35,133 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:35,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:35,135 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:35,135 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSED 2023-11-19T01:24:35,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-11-19T01:24:35,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 1.0310 sec 2023-11-19T01:24:35,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-11-19T01:24:35,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN in 1.0350 sec 2023-11-19T01:24:35,140 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357075140"}]},"ts":"1700357075140"} 2023-11-19T01:24:35,141 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:24:35,148 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:24:35,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0540 sec 2023-11-19T01:24:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:24:35,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357075210 (current time:1700357075210). 2023-11-19T01:24:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357075210 VERSION not specified, setting to 2 2023-11-19T01:24:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27784035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:35,224 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55888, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-11-19T01:24:35,228 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:24:35,229 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:35,232 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357075210 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:35,234 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:24:35,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:24:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:24:35,643 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:24:35,644 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:24:35,644 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:24:35,644 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:35,645 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/a11bba30e02649f4a831043ab98ed269] hfiles 2023-11-19T01:24:35,645 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/a11bba30e02649f4a831043ab98ed269 2023-11-19T01:24:35,646 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:35,648 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe42e354ed8a48c49c27d01261963df4] hfiles 2023-11-19T01:24:35,648 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:35,648 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8 2023-11-19T01:24:35,648 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:35,648 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe42e354ed8a48c49c27d01261963df4 2023-11-19T01:24:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:24:36,055 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:24:36,056 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:36,057 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:36,057 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357075210 2023-11-19T01:24:36,058 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357075210 2023-11-19T01:24:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:24:36,468 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:36,477 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:36,477 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357075210 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357075210 2023-11-19T01:24:36,479 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:36,479 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-11-19T01:24:36,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357075210 table=backup:system type=FLUSH ttl=0 } in 1.2540 sec 2023-11-19T01:24:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:24:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357071661 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:24:37,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:24:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-19T01:24:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-19T01:24:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-19T01:24:37,760 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-19T01:24:37,760 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-11-19T01:24:37,762 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:37,764 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357071661 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:37,764 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system 2023-11-19T01:24:37,767 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:24:37,767 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:37,774 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/a11bba30e02649f4a831043ab98ed269 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/a11bba30e02649f4a831043ab98ed269 2023-11-19T01:24:37,774 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:37,779 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe42e354ed8a48c49c27d01261963df4 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe42e354ed8a48c49c27d01261963df4 2023-11-19T01:24:37,779 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:37,779 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357071661 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:37,780 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357077780"}]},"ts":"9223372036854775807"} 2023-11-19T01:24:37,783 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:24:37,783 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:37,783 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357077781"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357077781"}]},"ts":"1700357077781"} 2023-11-19T01:24:37,784 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:37,784 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:24:37,784 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:37,784 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:24:37,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357071661 type: FLUSH version: 2 ttl: 0 ) in 441 msec 2023-11-19T01:24:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-19T01:24:37,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357075210" 2023-11-19T01:24:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357075210 2023-11-19T01:24:37,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:24:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:24:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:24:37,960 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:24:37,960 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357077960"}]},"ts":"1700357077960"} 2023-11-19T01:24:37,961 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:24:37,964 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:24:37,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN}] 2023-11-19T01:24:37,966 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN 2023-11-19T01:24:37,966 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:24:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:24:38,118 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:38,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:24:38,272 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:38,276 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(130): Open backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:38,276 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7254): Opening region: {ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:38,276 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:38,276 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,277 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(888): Instantiated backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:38,277 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7296): checking encryption for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,277 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7299): checking classloading for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,278 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,279 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName meta 2023-11-19T01:24:38,279 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:38,280 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:38,280 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,281 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName session 2023-11-19T01:24:38,281 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:38,289 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:38,294 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8 2023-11-19T01:24:38,299 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:38,299 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:38,300 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,302 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:24:38,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1076): writing seq id for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:38,304 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): Opened d634a0c0d7e9a4d882c1acc22f37a51c; next sequenceid=42; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10324406720, jitterRate=-0.03846469521522522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:24:38,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(992): Region open journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:38,306 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., pid=61, masterSystemTime=1700357078271 2023-11-19T01:24:38,308 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(403): Add compact mark for store d634a0c0d7e9a4d882c1acc22f37a51c:session, priority=-2147483648, current under compaction store size is 1 2023-11-19T01:24:38,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-19T01:24:38,309 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-11-19T01:24:38,311 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:38,311 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(158): Opened backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:38,311 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPEN, openSeqNum=42, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:38,312 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 15770 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-11-19T01:24:38,315 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.HStore(1534): d634a0c0d7e9a4d882c1acc22f37a51c/session is initiating minor compaction (all files) 2023-11-19T01:24:38,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-11-19T01:24:38,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 193 msec 2023-11-19T01:24:38,315 INFO [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of d634a0c0d7e9a4d882c1acc22f37a51c/session in backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:38,315 INFO [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8] into tmpdir=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp, totalSize=15.4 K 2023-11-19T01:24:38,316 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] compactions.Compactor(223): Compacting ddf84a84c0ff450a96ad47de731903b5, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1700357058983 2023-11-19T01:24:38,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=60, resume processing ppid=59 2023-11-19T01:24:38,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, ppid=59, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN in 351 msec 2023-11-19T01:24:38,317 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7670c7518979419f8bdd8eeb610c071b, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1700357064778 2023-11-19T01:24:38,317 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357078317"}]},"ts":"1700357078317"} 2023-11-19T01:24:38,317 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7796765c6c0d4267b772f182b8abd1e8, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1700357071538 2023-11-19T01:24:38,318 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:24:38,320 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:24:38,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; EnableTableProcedure table=backup:system in 365 msec 2023-11-19T01:24:38,338 INFO [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d634a0c0d7e9a4d882c1acc22f37a51c#session#compaction#8 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-11-19T01:24:38,339 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/f4968e8cb34142cc98c0bb6b58c30a8b is 284, key is session:backup_1700357058926/session:context/1700357064778/Put/seqid=0 2023-11-19T01:24:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:24:38,562 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:24:38,562 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:24:38,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:24:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:24:38,570 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:24:38,570 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-19T01:24:38,572 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1700357071642 2023-11-19T01:24:38,577 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:38,577 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:38,618 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupRepair(57): Running stage 3 2023-11-19T01:24:38,637 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43966bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:38,638 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:38,639 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47368, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:38,643 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:38,644 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55898, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:38,701 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:24:38,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357078706 (current time:1700357078706). 2023-11-19T01:24:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:24:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b23dea2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:38,719 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55902, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-19T01:24:38,724 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:24:38,725 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:38,728 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:38,732 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:38,753 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/f4968e8cb34142cc98c0bb6b58c30a8b as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/f4968e8cb34142cc98c0bb6b58c30a8b 2023-11-19T01:24:38,773 INFO [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in d634a0c0d7e9a4d882c1acc22f37a51c/session of d634a0c0d7e9a4d882c1acc22f37a51c into f4968e8cb34142cc98c0bb6b58c30a8b(size=5.6 K), total size for store is 5.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-11-19T01:24:38,773 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:38,773 INFO [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., storeName=d634a0c0d7e9a4d882c1acc22f37a51c/session, priority=13, startTime=1700357078308; duration=0sec 2023-11-19T01:24:38,774 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-19T01:24:38,774 DEBUG [RS:0;jenkins-hbase6:36757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d634a0c0d7e9a4d882c1acc22f37a51c:session 2023-11-19T01:24:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:24:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:24:39,148 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:24:39,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c}] 2023-11-19T01:24:39,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:39,301 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:39,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36757 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-11-19T01:24:39,302 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:39,303 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-11-19T01:24:39,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/8e1825f93557467a946439d7385e9abd is 284, key is session:backup_1700357071642/session:context/1700357078571/Put/seqid=0 2023-11-19T01:24:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:24:39,714 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/8e1825f93557467a946439d7385e9abd 2023-11-19T01:24:39,723 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/8e1825f93557467a946439d7385e9abd as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd 2023-11-19T01:24:39,729 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd, entries=2, sequenceid=48, filesize=5.2 K 2023-11-19T01:24:39,730 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 427ms, sequenceid=48, compaction requested=false 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. for snapshot_backup_system completed. 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/f4968e8cb34142cc98c0bb6b58c30a8b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd] hfiles 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/f4968e8cb34142cc98c0bb6b58c30a8b for snapshot=snapshot_backup_system 2023-11-19T01:24:39,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd for snapshot=snapshot_backup_system 2023-11-19T01:24:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:24:40,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:40,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-11-19T01:24:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster(4026): Remote procedure done, pid=63 2023-11-19T01:24:40,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:40,140 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:40,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-11-19T01:24:40,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c in 993 msec 2023-11-19T01:24:40,142 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:24:40,143 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:40,144 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:40,144 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:24:40,145 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:24:40,559 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:40,568 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:40,568 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:24:40,570 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:40,570 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-19T01:24:40,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8490 sec 2023-11-19T01:24:40,636 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:24:40,831 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(115): Backup backup_1700357078685 started at 1700357080831. 2023-11-19T01:24:40,833 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(119): Backup session backup_1700357078685 has been started. 2023-11-19T01:24:40,835 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-11-19T01:24:40,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:24:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-19T01:24:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:24:40,836 INFO [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:24:40,836 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:24:40,838 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:24:40,838 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:24:40,838 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:24:40,840 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,840 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:24:40,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:24:40,841 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:24:40,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:24:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:24:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:24:40,842 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:24:40,842 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:24:40,842 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-19T01:24:40,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:24:40,843 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:24:40,843 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:24:40,845 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:24:40,845 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:24:40,845 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,36757,1700357037569' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:24:40,848 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:40,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:24:40,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:24:40,848 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:24:40,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:24:40,849 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:24:40,849 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:40,849 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,850 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:24:40,850 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,36757,1700357037569' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:24:40,850 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6502864b[Count = 0] remaining members to acquire global barrier 2023-11-19T01:24:40,850 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:24:40,850 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:40,852 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:40,852 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:24:40,852 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:24:40,852 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:24:40,852 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:24:40,852 DEBUG [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,852 INFO [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357073860 highest: 1700357073860 on jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,852 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta:.meta(num 1700357073836) roll requested 2023-11-19T01:24:40,864 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357080853.meta, exclude list is [], retry=0 2023-11-19T01:24:40,867 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:24:40,870 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357073836.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357080853.meta 2023-11-19T01:24:40,870 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:24:40,871 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta.1700357073836.meta is not closed yet, will try archiving it next time 2023-11-19T01:24:40,871 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C36757%2C1700357037569:(num 1700357073860) roll requested 2023-11-19T01:24:40,882 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357080871, exclude list is [], retry=0 2023-11-19T01:24:40,886 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK] 2023-11-19T01:24:40,887 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357073860 with entries=13, filesize=3.42 KB; new WAL /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357080871 2023-11-19T01:24:40,888 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37467,DS-c9c75027-187c-468b-be8d-a2e13e078574,DISK]] 2023-11-19T01:24:40,888 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357073860 is not closed yet, will try archiving it next time 2023-11-19T01:24:40,893 DEBUG [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-19T01:24:40,893 INFO [rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357080871 on jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,900 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:24:40,900 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,36757,1700357037569' in zk 2023-11-19T01:24:40,904 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:24:40,904 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:24:40,904 DEBUG [member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:24:40,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:24:40,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:24:40,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:24:40,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:24:40,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:40,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:24:40,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:40,907 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,907 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,36757,1700357037569': 2023-11-19T01:24:40,907 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,36757,1700357037569' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:24:40,907 INFO [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:24:40,908 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:24:40,908 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:24:40,908 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:24:40,908 INFO [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:24:40,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:24:40,910 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:24:40,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:24:40,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:24:40,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:24:40,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:24:40,911 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:24:40,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:24:40,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:40,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:24:40,911 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:24:40,912 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:40,912 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:24:40,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:24:40,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:40,918 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:24:40,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:24:40,919 DEBUG [(jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:24:40,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:24:40,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:24:40,919 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:24:40,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:24:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:24:41,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } 2023-11-19T01:24:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357081023 (current time:1700357081023). 2023-11-19T01:24:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357081022_default_test-1700357047223 VERSION not specified, setting to 2 2023-11-19T01:24:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c639b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:41,035 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } 2023-11-19T01:24:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-19T01:24:41,039 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:24:41,040 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:41,043 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_1700357081022_default_test-1700357047223 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:41,046 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:24:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:24:41,454 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:24:41,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 9c7b67ce00e8024f201b8cb1555d84b3}] 2023-11-19T01:24:41,456 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:41,607 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:41,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36757 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-11-19T01:24:41,608 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:41,609 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 9c7b67ce00e8024f201b8cb1555d84b3 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:24:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:24:41,650 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/.tmp/f/d2c0589614434c46a1fe7601eaf15e17 is 37, key is row10/f:q1/1700357049861/Put/seqid=0 2023-11-19T01:24:41,655 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/.tmp/f/d2c0589614434c46a1fe7601eaf15e17 2023-11-19T01:24:41,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/.tmp/f/d2c0589614434c46a1fe7601eaf15e17 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/f/d2c0589614434c46a1fe7601eaf15e17 2023-11-19T01:24:41,670 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/f/d2c0589614434c46a1fe7601eaf15e17, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:24:41,671 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 9c7b67ce00e8024f201b8cb1555d84b3 in 62ms, sequenceid=103, compaction requested=false 2023-11-19T01:24:41,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 9c7b67ce00e8024f201b8cb1555d84b3: 2023-11-19T01:24:41,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. for snapshot_1700357081022_default_test-1700357047223 completed. 2023-11-19T01:24:41,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3.' region-info for snapshot=snapshot_1700357081022_default_test-1700357047223 2023-11-19T01:24:41,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:24:41,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/f/d2c0589614434c46a1fe7601eaf15e17] hfiles 2023-11-19T01:24:41,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/f/d2c0589614434c46a1fe7601eaf15e17 for snapshot=snapshot_1700357081022_default_test-1700357047223 2023-11-19T01:24:42,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:42,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-11-19T01:24:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster(4026): Remote procedure done, pid=65 2023-11-19T01:24:42,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357081022_default_test-1700357047223 on region 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:42,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:42,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-11-19T01:24:42,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 9c7b67ce00e8024f201b8cb1555d84b3 in 627 msec 2023-11-19T01:24:42,083 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:24:42,084 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:42,084 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:42,085 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357081022_default_test-1700357047223 2023-11-19T01:24:42,086 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_1700357081022_default_test-1700357047223 2023-11-19T01:24:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:24:42,497 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:42,503 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:42,503 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_1700357081022_default_test-1700357047223 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/snapshot_1700357081022_default_test-1700357047223 2023-11-19T01:24:42,504 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:42,504 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-19T01:24:42,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1700357081022_default_test-1700357047223 table=test-1700357047223 type=FLUSH ttl=0 } in 1.4680 sec 2023-11-19T01:24:42,953 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-19T01:24:42,953 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-19T01:24:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:24:43,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } 2023-11-19T01:24:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357083148 (current time:1700357083148). 2023-11-19T01:24:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357083147_ns2_test-17003570472231 VERSION not specified, setting to 2 2023-11-19T01:24:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5b12e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:43,159 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35312, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } 2023-11-19T01:24:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-11-19T01:24:43,164 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:24:43,164 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:43,167 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_1700357083147_ns2_test-17003570472231 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:43,170 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:24:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:24:43,580 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:24:43,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure b3a631670a013c3c95d5f63d2f831b4e}] 2023-11-19T01:24:43,583 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:43,734 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:43,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36757 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-11-19T01:24:43,735 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:43,736 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing b3a631670a013c3c95d5f63d2f831b4e 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:24:43,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/.tmp/f/be89479bd5c442e7bdf630c7a528ff72 is 37, key is row10/f:q1/1700357052091/Put/seqid=0 2023-11-19T01:24:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:24:44,156 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/.tmp/f/be89479bd5c442e7bdf630c7a528ff72 2023-11-19T01:24:44,177 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/.tmp/f/be89479bd5c442e7bdf630c7a528ff72 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/f/be89479bd5c442e7bdf630c7a528ff72 2023-11-19T01:24:44,184 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/f/be89479bd5c442e7bdf630c7a528ff72, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:24:44,185 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for b3a631670a013c3c95d5f63d2f831b4e in 449ms, sequenceid=103, compaction requested=false 2023-11-19T01:24:44,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for b3a631670a013c3c95d5f63d2f831b4e: 2023-11-19T01:24:44,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. for snapshot_1700357083147_ns2_test-17003570472231 completed. 2023-11-19T01:24:44,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.' region-info for snapshot=snapshot_1700357083147_ns2_test-17003570472231 2023-11-19T01:24:44,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:24:44,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/f/be89479bd5c442e7bdf630c7a528ff72] hfiles 2023-11-19T01:24:44,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/f/be89479bd5c442e7bdf630c7a528ff72 for snapshot=snapshot_1700357083147_ns2_test-17003570472231 2023-11-19T01:24:44,210 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:44,210 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-11-19T01:24:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster(4026): Remote procedure done, pid=67 2023-11-19T01:24:44,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357083147_ns2_test-17003570472231 on region b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:44,212 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:44,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-11-19T01:24:44,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure b3a631670a013c3c95d5f63d2f831b4e in 633 msec 2023-11-19T01:24:44,215 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:24:44,215 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:44,216 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:44,216 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357083147_ns2_test-17003570472231 2023-11-19T01:24:44,216 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_1700357083147_ns2_test-17003570472231 2023-11-19T01:24:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:24:44,635 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:44,641 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:44,642 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/snapshot_1700357083147_ns2_test-17003570472231 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/snapshot_1700357083147_ns2_test-17003570472231 2023-11-19T01:24:44,643 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:44,643 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-11-19T01:24:44,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357083147_ns2_test-17003570472231 table=ns2:test-17003570472231 type=FLUSH ttl=0 } in 1.4830 sec 2023-11-19T01:24:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 Backup session finished. Status: FAILURE 2023-11-19T01:24:45,270 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:45,270 ERROR [Listener at jenkins-hbase6.apache.org/34947 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] ... 32 more 2023-11-19T01:24:45,278 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b183fd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:45,284 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a07b0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:45,285 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:24:45,286 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44098, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:24:45,293 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:45,295 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35326, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1700357078685,startts=1700357080831,failedts=1700357085313,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1700357078685 2023-11-19T01:24:45,313 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:24:45,314 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:24:45,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:24:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:24:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:24:45,329 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357085329"}]},"ts":"1700357085329"} 2023-11-19T01:24:45,331 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:24:45,334 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:24:45,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN}] 2023-11-19T01:24:45,337 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN 2023-11-19T01:24:45,337 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:45,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:45,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:24:45,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-19T01:24:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:24:45,490 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:45,491 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:45,491 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing d634a0c0d7e9a4d882c1acc22f37a51c, disabling compactions & flushes 2023-11-19T01:24:45,491 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:45,491 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:45,491 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. after waiting 0 ms 2023-11-19T01:24:45,491 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:45,491 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-11-19T01:24:45,505 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/8dbb4a0e385243fb81d26f95450d9c25 is 116, key is rslogts:hdfs://localhost:45331/backupUT\x00jenkins-hbase6.apache.org:36757/meta:rs-log-ts/1700357080900/Put/seqid=0 2023-11-19T01:24:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:24:45,910 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/8dbb4a0e385243fb81d26f95450d9c25 2023-11-19T01:24:45,920 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/fe200ae2535449089ea06261cfc9a193 is 218, key is session:backup_1700357078685/session:context/1700357080832/Put/seqid=0 2023-11-19T01:24:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:24:46,324 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/fe200ae2535449089ea06261cfc9a193 2023-11-19T01:24:46,331 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/meta/8dbb4a0e385243fb81d26f95450d9c25 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/8dbb4a0e385243fb81d26f95450d9c25 2023-11-19T01:24:46,338 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/8dbb4a0e385243fb81d26f95450d9c25, entries=2, sequenceid=54, filesize=5.3 K 2023-11-19T01:24:46,342 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/fe200ae2535449089ea06261cfc9a193 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe200ae2535449089ea06261cfc9a193 2023-11-19T01:24:46,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe200ae2535449089ea06261cfc9a193, entries=1, sequenceid=54, filesize=5.2 K 2023-11-19T01:24:46,349 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 858ms, sequenceid=54, compaction requested=true 2023-11-19T01:24:46,357 DEBUG [StoreCloser-backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8] to archive 2023-11-19T01:24:46,358 DEBUG [StoreCloser-backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-11-19T01:24:46,363 DEBUG [StoreCloser-backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/ddf84a84c0ff450a96ad47de731903b5 2023-11-19T01:24:46,365 DEBUG [StoreCloser-backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7670c7518979419f8bdd8eeb610c071b 2023-11-19T01:24:46,367 DEBUG [StoreCloser-backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/7796765c6c0d4267b772f182b8abd1e8 2023-11-19T01:24:46,418 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/recovered.edits/57.seqid, newMaxSeqId=57, maxSeqId=41 2023-11-19T01:24:46,419 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:46,420 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:46,420 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:46,421 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:46,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=CLOSED 2023-11-19T01:24:46,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-11-19T01:24:46,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 1.0840 sec 2023-11-19T01:24:46,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-11-19T01:24:46,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, UNASSIGN in 1.0890 sec 2023-11-19T01:24:46,427 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357086426"}]},"ts":"1700357086426"} 2023-11-19T01:24:46,428 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:24:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:24:46,434 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:24:46,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; DisableTableProcedure table=backup:system in 1.1090 sec 2023-11-19T01:24:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:24:47,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357087449 (current time:1700357087449). 2023-11-19T01:24:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:24:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357087448 VERSION not specified, setting to 2 2023-11-19T01:24:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e7810f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:24:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:24:47,461 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:24:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:24:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-11-19T01:24:47,465 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:24:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:24:47,466 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:24:47,469 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot, current working dir path: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357087448 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:24:47,471 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:24:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:24:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:24:47,877 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:24:47,878 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:24:47,878 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:24:47,878 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:47,879 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/8dbb4a0e385243fb81d26f95450d9c25] hfiles 2023-11-19T01:24:47,879 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/8dbb4a0e385243fb81d26f95450d9c25 2023-11-19T01:24:47,879 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:47,881 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/f4968e8cb34142cc98c0bb6b58c30a8b, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe200ae2535449089ea06261cfc9a193] hfiles 2023-11-19T01:24:47,881 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd 2023-11-19T01:24:47,881 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/f4968e8cb34142cc98c0bb6b58c30a8b 2023-11-19T01:24:47,881 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe200ae2535449089ea06261cfc9a193 2023-11-19T01:24:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:24:48,286 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:24:48,288 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:24:48,288 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:24:48,289 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357087448 2023-11-19T01:24:48,289 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357087448 2023-11-19T01:24:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:24:48,702 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:24:48,710 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:24:48,713 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357087448 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357087448 2023-11-19T01:24:48,715 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:24:48,715 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-11-19T01:24:48,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357087448 table=backup:system type=FLUSH ttl=0 } in 1.2520 sec 2023-11-19T01:24:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:24:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357078706 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:24:49,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:24:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:24:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:24:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:24:49,991 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-11-19T01:24:49,991 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-11-19T01:24:49,992 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:24:49,995 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357078706 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:49,995 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system 2023-11-19T01:24:49,998 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:24:49,998 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,006 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/8dbb4a0e385243fb81d26f95450d9c25 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/meta/8dbb4a0e385243fb81d26f95450d9c25 2023-11-19T01:24:50,006 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:50,009 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe200ae2535449089ea06261cfc9a193 to hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/archive/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/fe200ae2535449089ea06261cfc9a193 2023-11-19T01:24:50,009 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:50,009 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357078706 type: FLUSH version: 2 ttl: 0 2023-11-19T01:24:50,010 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357090010"}]},"ts":"9223372036854775807"} 2023-11-19T01:24:50,012 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:24:50,012 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:50,012 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357090011"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357090011"}]},"ts":"1700357090011"} 2023-11-19T01:24:50,013 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:24:50,013 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:24:50,013 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:24:50,013 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:24:50,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357078706 type: FLUSH version: 2 ttl: 0 ) in 436 msec 2023-11-19T01:24:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:24:50,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357087448" 2023-11-19T01:24:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357087448 2023-11-19T01:24:50,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:24:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] procedure2.ProcedureExecutor(1032): Stored pid=73, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:24:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2023-11-19T01:24:50,194 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:24:50,194 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357090194"}]},"ts":"1700357090194"} 2023-11-19T01:24:50,195 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:24:50,198 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:24:50,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN}] 2023-11-19T01:24:50,199 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN 2023-11-19T01:24:50,200 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:24:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2023-11-19T01:24:50,352 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:50,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569}] 2023-11-19T01:24:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2023-11-19T01:24:50,505 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:50,509 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(130): Open backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:50,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7254): Opening region: {ENCODED => d634a0c0d7e9a4d882c1acc22f37a51c, NAME => 'backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:24:50,509 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:24:50,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(888): Instantiated backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:24:50,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7296): checking encryption for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7299): checking classloading for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,511 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,512 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName meta 2023-11-19T01:24:50,512 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:50,512 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:50,512 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,513 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d634a0c0d7e9a4d882c1acc22f37a51c columnFamilyName session 2023-11-19T01:24:50,513 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:24:50,519 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/8e1825f93557467a946439d7385e9abd 2023-11-19T01:24:50,523 DEBUG [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/f4968e8cb34142cc98c0bb6b58c30a8b 2023-11-19T01:24:50,523 INFO [StoreOpener-d634a0c0d7e9a4d882c1acc22f37a51c-1 {}] regionserver.HStore(324): Store=d634a0c0d7e9a4d882c1acc22f37a51c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:24:50,524 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,525 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,526 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:24:50,527 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1076): writing seq id for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,528 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): Opened d634a0c0d7e9a4d882c1acc22f37a51c; next sequenceid=58; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11946148160, jitterRate=0.11257174611091614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:24:50,528 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(992): Region open journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:50,529 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., pid=75, masterSystemTime=1700357090505 2023-11-19T01:24:50,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:50,531 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(158): Opened backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:50,531 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=d634a0c0d7e9a4d882c1acc22f37a51c, regionState=OPEN, openSeqNum=58, regionLocation=jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:50,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-11-19T01:24:50,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; OpenRegionProcedure d634a0c0d7e9a4d882c1acc22f37a51c, server=jenkins-hbase6.apache.org,36757,1700357037569 in 178 msec 2023-11-19T01:24:50,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-11-19T01:24:50,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=d634a0c0d7e9a4d882c1acc22f37a51c, ASSIGN in 336 msec 2023-11-19T01:24:50,536 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357090536"}]},"ts":"1700357090536"} 2023-11-19T01:24:50,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:24:50,540 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:24:50,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, state=SUCCESS; EnableTableProcedure table=backup:system in 350 msec 2023-11-19T01:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2023-11-19T01:24:50,797 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:24:50,797 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:24:50,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:24:50,808 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:24:50,808 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-19T01:24:50,810 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] impl.BackupSystemTable(657): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1700357078685 2023-11-19T01:24:50,815 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:50,815 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:50,875 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.ResourceChecker(175): after: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=609 (was 584) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase6:36757-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3488e93c-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61923@0x6f975356-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1683785708_16 at /127.0.0.1:57498 [Receiving block BP-6675545-172.31.2.20-1700357030750:blk_1073741888_1064] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61923@0x6f975356-SendThread(127.0.0.1:61923) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: member: 'jenkins-hbase6.apache.org,36757,1700357037569' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61923@0x6f975356 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$538/2116380524.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3488e93c-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-77465695_16 at /127.0.0.1:57634 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-77465695_16 at /127.0.0.1:57646 [Waiting for operation #13] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3488e93c-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-77465695_16 at /127.0.0.1:57590 [Waiting for operation #21] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6ab7bfea-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3488e93c-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-6675545-172.31.2.20-1700357030750:blk_1073741887_1063, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6ab7bfea-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61923@0x501c81ea-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: hconnection-0x6ab7bfea-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3488e93c-shared-pool-27 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-6675545-172.31.2.20-1700357030750:blk_1073741888_1064, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61923@0x501c81ea-SendThread(127.0.0.1:61923) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61923@0x501c81ea java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$538/2116380524.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,36757,1700357037569)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6ab7bfea-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6ab7bfea-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3488e93c-shared-pool-26 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1683785708_16 at /127.0.0.1:57496 [Receiving block BP-6675545-172.31.2.20-1700357030750:blk_1073741887_1063] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase6.apache.org,34089,1700357034167)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=722 (was 689) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=194 (was 204), ProcessCount=169 (was 169), AvailableMemoryMB=4176 (was 3834) - AvailableMemoryMB LEAK? - 2023-11-19T01:24:50,875 WARN [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.ResourceChecker(130): Thread=609 is superior to 500 2023-11-19T01:24:50,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357081022_default_test-1700357047223" 2023-11-19T01:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357081022_default_test-1700357047223 2023-11-19T01:24:50,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357083147_ns2_test-17003570472231" 2023-11-19T01:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34089 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357083147_ns2_test-17003570472231 2023-11-19T01:24:50,892 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-19T01:24:50,893 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:50,893 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-19T01:24:50,893 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] util.JVMClusterUtil(257): Found active master hash=8050731, stopped=false 2023-11-19T01:24:50,893 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:24:50,896 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-19T01:24:50,897 DEBUG [Listener at jenkins-hbase6.apache.org/34947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:50,897 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,36757,1700357037569' ***** 2023-11-19T01:24:50,897 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-19T01:24:50,897 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:24:50,912 INFO [RS:0;jenkins-hbase6:36757 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@7b99e2c7{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:24:50,916 INFO [RS:0;jenkins-hbase6:36757 {}] server.AbstractConnector(383): Stopped ServerConnector@7cdacb0f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:24:50,917 INFO [RS:0;jenkins-hbase6:36757 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:24:50,917 INFO [RS:0;jenkins-hbase6:36757 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@45f7ed9d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:24:50,918 INFO [RS:0;jenkins-hbase6:36757 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@539fdaa1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.log.dir/,STOPPED} 2023-11-19T01:24:50,919 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-19T01:24:50,920 INFO [regionserver/jenkins-hbase6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:24:50,920 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-19T01:24:50,920 INFO [RS:0;jenkins-hbase6:36757 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for cd1564ff9977411d21e3ea8e55f13843 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for 4350ab61ef4b67e9a8a8417c991edc1d 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for 9c7b67ce00e8024f201b8cb1555d84b3 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for bf7d9872f21a90dc504641c20f40847a 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for 5859bbc8b279d13e33306d9bde605ca2 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:50,921 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,36757,1700357037569 2023-11-19T01:24:50,921 DEBUG [RS:0;jenkins-hbase6:36757 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:50,922 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-19T01:24:50,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing cd1564ff9977411d21e3ea8e55f13843, disabling compactions & flushes 2023-11-19T01:24:50,922 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-19T01:24:50,922 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-19T01:24:50,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:50,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:50,922 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-19T01:24:50,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. after waiting 0 ms 2023-11-19T01:24:50,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:50,922 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-19T01:24:50,922 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1583): Online Regions={cd1564ff9977411d21e3ea8e55f13843=ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843., 4350ab61ef4b67e9a8a8417c991edc1d=ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d., 9c7b67ce00e8024f201b8cb1555d84b3=test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3., 1588230740=hbase:meta,,1.1588230740, bf7d9872f21a90dc504641c20f40847a=backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a., d634a0c0d7e9a4d882c1acc22f37a51c=backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c., 5859bbc8b279d13e33306d9bde605ca2=hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2., b3a631670a013c3c95d5f63d2f831b4e=ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e.} 2023-11-19T01:24:50,923 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:24:50,923 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:24:50,923 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:24:50,923 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:24:50,923 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:24:50,923 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=17.20 KB heapSize=30.02 KB 2023-11-19T01:24:50,924 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 4350ab61ef4b67e9a8a8417c991edc1d, 5859bbc8b279d13e33306d9bde605ca2, 9c7b67ce00e8024f201b8cb1555d84b3, b3a631670a013c3c95d5f63d2f831b4e, bf7d9872f21a90dc504641c20f40847a, cd1564ff9977411d21e3ea8e55f13843, d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:50,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns4/test-17003570472233/cd1564ff9977411d21e3ea8e55f13843/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:24:50,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:50,930 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:50,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for cd1564ff9977411d21e3ea8e55f13843: 2023-11-19T01:24:50,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17003570472233,,1700357054288.cd1564ff9977411d21e3ea8e55f13843. 2023-11-19T01:24:50,931 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4350ab61ef4b67e9a8a8417c991edc1d, disabling compactions & flushes 2023-11-19T01:24:50,931 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:50,931 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:50,931 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. after waiting 0 ms 2023-11-19T01:24:50,931 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:50,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns3/test-17003570472232/4350ab61ef4b67e9a8a8417c991edc1d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:24:50,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:50,936 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4350ab61ef4b67e9a8a8417c991edc1d: 2023-11-19T01:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17003570472232,,1700357052162.4350ab61ef4b67e9a8a8417c991edc1d. 2023-11-19T01:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9c7b67ce00e8024f201b8cb1555d84b3, disabling compactions & flushes 2023-11-19T01:24:50,936 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. after waiting 0 ms 2023-11-19T01:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:50,941 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/default/test-1700357047223/9c7b67ce00e8024f201b8cb1555d84b3/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-19T01:24:50,942 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:50,943 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9c7b67ce00e8024f201b8cb1555d84b3: 2023-11-19T01:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1700357047223,,1700357047708.9c7b67ce00e8024f201b8cb1555d84b3. 2023-11-19T01:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing bf7d9872f21a90dc504641c20f40847a, disabling compactions & flushes 2023-11-19T01:24:50,943 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. after waiting 0 ms 2023-11-19T01:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:50,947 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system_bulk/bf7d9872f21a90dc504641c20f40847a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:24:50,948 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:50,948 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:50,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for bf7d9872f21a90dc504641c20f40847a: 2023-11-19T01:24:50,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1700357057717.bf7d9872f21a90dc504641c20f40847a. 2023-11-19T01:24:50,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d634a0c0d7e9a4d882c1acc22f37a51c, disabling compactions & flushes 2023-11-19T01:24:50,949 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:50,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:50,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. after waiting 0 ms 2023-11-19T01:24:50,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:50,949 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing d634a0c0d7e9a4d882c1acc22f37a51c 2/2 column families, dataSize=324 B heapSize=984 B 2023-11-19T01:24:50,953 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/.tmp/info/6d81b126c8b1486abb1e9f172b05f2df is 159, key is ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e./info:regioninfo/1700357051156/Put/seqid=0 2023-11-19T01:24:50,966 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/1bb056531a6c4fe388ebb683e02fb5ba is 284, key is session:backup_1700357078685/session:context/1700357090809/Put/seqid=0 2023-11-19T01:24:51,124 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 5859bbc8b279d13e33306d9bde605ca2, b3a631670a013c3c95d5f63d2f831b4e, d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:51,324 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 5859bbc8b279d13e33306d9bde605ca2, b3a631670a013c3c95d5f63d2f831b4e, d634a0c0d7e9a4d882c1acc22f37a51c 2023-11-19T01:24:51,361 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.80 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/.tmp/info/6d81b126c8b1486abb1e9f172b05f2df 2023-11-19T01:24:51,370 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/1bb056531a6c4fe388ebb683e02fb5ba 2023-11-19T01:24:51,377 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/.tmp/session/1bb056531a6c4fe388ebb683e02fb5ba as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/1bb056531a6c4fe388ebb683e02fb5ba 2023-11-19T01:24:51,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/.tmp/table/1512c380691047b192fe315661acfdd0 is 59, key is ns2:test-17003570472231/table:state/1700357051166/Put/seqid=0 2023-11-19T01:24:51,382 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/session/1bb056531a6c4fe388ebb683e02fb5ba, entries=2, sequenceid=62, filesize=5.2 K 2023-11-19T01:24:51,383 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~712 B/712, currentSize=0 B/0 for d634a0c0d7e9a4d882c1acc22f37a51c in 434ms, sequenceid=62, compaction requested=true 2023-11-19T01:24:51,383 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:24:51,395 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/backup/system/d634a0c0d7e9a4d882c1acc22f37a51c/recovered.edits/65.seqid, newMaxSeqId=65, maxSeqId=57 2023-11-19T01:24:51,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:51,396 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:51,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d634a0c0d7e9a4d882c1acc22f37a51c: 2023-11-19T01:24:51,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1700357055588.d634a0c0d7e9a4d882c1acc22f37a51c. 2023-11-19T01:24:51,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 5859bbc8b279d13e33306d9bde605ca2, disabling compactions & flushes 2023-11-19T01:24:51,397 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:51,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:51,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. after waiting 0 ms 2023-11-19T01:24:51,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:51,397 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 5859bbc8b279d13e33306d9bde605ca2 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-19T01:24:51,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/.tmp/info/4a5c9572b9b2404db057b080e322699e is 45, key is default/info:d/1700357042431/Put/seqid=0 2023-11-19T01:24:51,524 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 5859bbc8b279d13e33306d9bde605ca2, b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:51,586 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-19T01:24:51,586 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-19T01:24:51,725 DEBUG [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 5859bbc8b279d13e33306d9bde605ca2, b3a631670a013c3c95d5f63d2f831b4e 2023-11-19T01:24:51,794 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/.tmp/table/1512c380691047b192fe315661acfdd0 2023-11-19T01:24:51,801 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/.tmp/info/6d81b126c8b1486abb1e9f172b05f2df as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/info/6d81b126c8b1486abb1e9f172b05f2df 2023-11-19T01:24:51,807 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/info/6d81b126c8b1486abb1e9f172b05f2df, entries=74, sequenceid=79, filesize=14.4 K 2023-11-19T01:24:51,807 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/.tmp/table/1512c380691047b192fe315661acfdd0 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/table/1512c380691047b192fe315661acfdd0 2023-11-19T01:24:51,813 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/table/1512c380691047b192fe315661acfdd0, entries=15, sequenceid=79, filesize=5.8 K 2023-11-19T01:24:51,814 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~17.20 KB/17610, heapSize ~29.73 KB/30448, currentSize=0 B/0 for 1588230740 in 891ms, sequenceid=79, compaction requested=false 2023-11-19T01:24:51,823 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/.tmp/info/4a5c9572b9b2404db057b080e322699e 2023-11-19T01:24:51,830 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=1 2023-11-19T01:24:51,830 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:51,831 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:24:51,831 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:24:51,831 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:24:51,831 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-19T01:24:51,832 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/.tmp/info/4a5c9572b9b2404db057b080e322699e as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/info/4a5c9572b9b2404db057b080e322699e 2023-11-19T01:24:51,837 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/info/4a5c9572b9b2404db057b080e322699e, entries=7, sequenceid=11, filesize=5.1 K 2023-11-19T01:24:51,838 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 5859bbc8b279d13e33306d9bde605ca2 in 441ms, sequenceid=11, compaction requested=false 2023-11-19T01:24:51,848 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/hbase/namespace/5859bbc8b279d13e33306d9bde605ca2/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-19T01:24:51,849 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:51,849 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:51,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 5859bbc8b279d13e33306d9bde605ca2: 2023-11-19T01:24:51,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1700357041053.5859bbc8b279d13e33306d9bde605ca2. 2023-11-19T01:24:51,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b3a631670a013c3c95d5f63d2f831b4e, disabling compactions & flushes 2023-11-19T01:24:51,850 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:51,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:51,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. after waiting 0 ms 2023-11-19T01:24:51,850 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:51,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/data/ns2/test-17003570472231/b3a631670a013c3c95d5f63d2f831b4e/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-19T01:24:51,855 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:24:51,856 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:51,856 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b3a631670a013c3c95d5f63d2f831b4e: 2023-11-19T01:24:51,856 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17003570472231,,1700357049961.b3a631670a013c3c95d5f63d2f831b4e. 2023-11-19T01:24:51,925 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,36757,1700357037569; all regions closed. 2023-11-19T01:24:51,935 DEBUG [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/oldWALs 2023-11-19T01:24:51,935 INFO [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C36757%2C1700357037569.meta:.meta(num 1700357080853) 2023-11-19T01:24:51,937 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/WALs/jenkins-hbase6.apache.org,36757,1700357037569/jenkins-hbase6.apache.org%2C36757%2C1700357037569.1700357080871 not finished, retry = 0 2023-11-19T01:24:51,962 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:24:52,043 DEBUG [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/oldWALs 2023-11-19T01:24:52,043 INFO [RS:0;jenkins-hbase6:36757 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C36757%2C1700357037569:(num 1700357080871) 2023-11-19T01:24:52,043 DEBUG [RS:0;jenkins-hbase6:36757 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:52,043 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:24:52,043 INFO [RS:0;jenkins-hbase6:36757 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-19T01:24:52,044 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:24:52,046 INFO [RS:0;jenkins-hbase6:36757 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:36757 2023-11-19T01:24:52,056 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase6.apache.org,36757,1700357037569] 2023-11-19T01:24:52,056 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase6.apache.org,36757,1700357037569; numProcessing=1 2023-11-19T01:24:52,057 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase6.apache.org,36757,1700357037569 expired; onlineServers=0 2023-11-19T01:24:52,057 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,34089,1700357034167' ***** 2023-11-19T01:24:52,057 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-19T01:24:52,058 DEBUG [M:0;jenkins-hbase6:34089 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229aad74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:24:52,058 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:24:52,105 INFO [M:0;jenkins-hbase6:34089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@661f9dfa{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:24:52,106 INFO [M:0;jenkins-hbase6:34089 {}] server.AbstractConnector(383): Stopped ServerConnector@76b72c0e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:24:52,106 INFO [M:0;jenkins-hbase6:34089 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:24:52,107 INFO [M:0;jenkins-hbase6:34089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2cfbfc75{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:24:52,108 INFO [M:0;jenkins-hbase6:34089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@b7ce79c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/hadoop.log.dir/,STOPPED} 2023-11-19T01:24:52,108 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,34089,1700357034167 2023-11-19T01:24:52,108 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,34089,1700357034167; all regions closed. 2023-11-19T01:24:52,108 DEBUG [M:0;jenkins-hbase6:34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:52,108 INFO [M:0;jenkins-hbase6:34089 {}] master.HMaster(1666): Stopping master jetty server 2023-11-19T01:24:52,109 INFO [M:0;jenkins-hbase6:34089 {}] server.AbstractConnector(383): Stopped ServerConnector@601759f1{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:24:52,109 DEBUG [M:0;jenkins-hbase6:34089 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-19T01:24:52,110 INFO [M:0;jenkins-hbase6:34089 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-19T01:24:52,110 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-19T01:24:52,110 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357039389 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357039389,5,FailOnTimeoutGroup] 2023-11-19T01:24:52,110 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357039389 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357039389,5,FailOnTimeoutGroup] 2023-11-19T01:24:52,110 DEBUG [M:0;jenkins-hbase6:34089 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:24:52,110 DEBUG [M:0;jenkins-hbase6:34089 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-19T01:24:52,110 INFO [M:0;jenkins-hbase6:34089 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase6:0 had [] on shutdown 2023-11-19T01:24:52,110 DEBUG [M:0;jenkins-hbase6:34089 {}] master.HMaster(1689): Stopping service threads 2023-11-19T01:24:52,110 INFO [M:0;jenkins-hbase6:34089 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-19T01:24:52,111 ERROR [M:0;jenkins-hbase6:34089 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-19T01:24:52,111 INFO [M:0;jenkins-hbase6:34089 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-19T01:24:52,111 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-19T01:24:52,112 WARN [M:0;jenkins-hbase6:34089 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-19T01:24:52,112 INFO [M:0;jenkins-hbase6:34089 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-19T01:24:52,113 INFO [M:0;jenkins-hbase6:34089 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-19T01:24:52,113 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:24:52,128 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:24:52,128 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:24:52,128 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:24:52,128 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:24:52,129 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=285.07 KB heapSize=342.16 KB 2023-11-19T01:24:52,146 DEBUG [M:0;jenkins-hbase6:34089 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/57b7ede30f88435290722cc0c75f308a is 88, key is hbase:meta,,1/info:sn/1700357040426/Put/seqid=0 2023-11-19T01:24:52,154 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/57b7ede30f88435290722cc0c75f308a 2023-11-19T01:24:52,156 INFO [RS:0;jenkins-hbase6:36757 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,36757,1700357037569; zookeeper connection closed. 2023-11-19T01:24:52,156 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54a08d1c {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54a08d1c 2023-11-19T01:24:52,158 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-19T01:24:52,180 DEBUG [M:0;jenkins-hbase6:34089 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/899f2f41dde044ed9d17bfdcab5165e3 is 1037, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1700357064046/Put/seqid=0 2023-11-19T01:24:52,585 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=284.48 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/899f2f41dde044ed9d17bfdcab5165e3 2023-11-19T01:24:52,591 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 899f2f41dde044ed9d17bfdcab5165e3 2023-11-19T01:24:52,606 DEBUG [M:0;jenkins-hbase6:34089 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dcbb4b040e304a7c98d582335e925a26 is 82, key is jenkins-hbase6.apache.org,36757,1700357037569/rs:state/1700357039467/Put/seqid=0 2023-11-19T01:24:53,011 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dcbb4b040e304a7c98d582335e925a26 2023-11-19T01:24:53,018 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/57b7ede30f88435290722cc0c75f308a as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/57b7ede30f88435290722cc0c75f308a 2023-11-19T01:24:53,023 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/57b7ede30f88435290722cc0c75f308a, entries=8, sequenceid=679, filesize=5.5 K 2023-11-19T01:24:53,024 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/899f2f41dde044ed9d17bfdcab5165e3 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/899f2f41dde044ed9d17bfdcab5165e3 2023-11-19T01:24:53,028 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 899f2f41dde044ed9d17bfdcab5165e3 2023-11-19T01:24:53,028 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/899f2f41dde044ed9d17bfdcab5165e3, entries=75, sequenceid=679, filesize=23.3 K 2023-11-19T01:24:53,029 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dcbb4b040e304a7c98d582335e925a26 as hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dcbb4b040e304a7c98d582335e925a26 2023-11-19T01:24:53,034 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45331/user/jenkins/test-data/6d8e4a8e-a754-1392-a777-3fc51ef83132/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dcbb4b040e304a7c98d582335e925a26, entries=1, sequenceid=679, filesize=5.1 K 2023-11-19T01:24:53,034 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(3022): Finished flush of dataSize ~285.07 KB/291914, heapSize ~341.87 KB/350072, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 906ms, sequenceid=679, compaction requested=false 2023-11-19T01:24:53,040 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:24:53,040 DEBUG [M:0;jenkins-hbase6:34089 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:24:53,043 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:24:53,044 INFO [M:0;jenkins-hbase6:34089 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-19T01:24:53,044 INFO [M:0;jenkins-hbase6:34089 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-19T01:24:53,044 INFO [M:0;jenkins-hbase6:34089 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:34089 2023-11-19T01:24:53,149 INFO [M:0;jenkins-hbase6:34089 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,34089,1700357034167; zookeeper connection closed. 2023-11-19T01:24:53,151 WARN [Listener at jenkins-hbase6.apache.org/34947 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:24:53,156 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:24:53,262 WARN [BP-6675545-172.31.2.20-1700357030750 heartbeating to localhost/127.0.0.1:45331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:24:53,262 WARN [BP-6675545-172.31.2.20-1700357030750 heartbeating to localhost/127.0.0.1:45331 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-6675545-172.31.2.20-1700357030750 (Datanode Uuid 0182dfdc-c62a-4f7e-a76c-e909395d0474) service to localhost/127.0.0.1:45331 2023-11-19T01:24:53,264 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/dfs/data/data1/current/BP-6675545-172.31.2.20-1700357030750 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:24:53,264 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/1be343c0-2cdd-c786-0221-4ed2cd060167/cluster_2f10e323-5ba2-9f9d-4dce-82ba057b229a/dfs/data/data2/current/BP-6675545-172.31.2.20-1700357030750 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:24:53,294 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:24:53,342 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-19T01:24:53,342 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-19T01:24:53,352 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:24:53,473 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:24:53,587 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:24:53,587 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:24:53,692 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-19T01:24:53,697 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-19T01:24:53,699 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:24:53,708 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:24:53,811 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:24:53,811 INFO [Listener at jenkins-hbase6.apache.org/34947 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped