2023-11-12 09:28:28,849 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da 2023-11-12 09:28:28,862 main DEBUG Took 0.010929 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-12 09:28:28,863 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-12 09:28:28,863 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-12 09:28:28,864 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-12 09:28:28,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,877 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-12 09:28:28,894 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,896 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,897 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,899 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,900 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,901 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,902 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,902 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,903 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,904 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,906 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:28:28,907 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,907 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-12 09:28:28,910 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:28:28,912 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-12 09:28:28,914 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-12 09:28:28,914 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-12 09:28:28,915 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-12 09:28:28,916 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-12 09:28:28,929 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-12 09:28:28,932 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-12 09:28:28,933 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-12 09:28:28,934 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-12 09:28:28,934 main DEBUG createAppenders(={Console}) 2023-11-12 09:28:28,935 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da initialized 2023-11-12 09:28:28,935 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da 2023-11-12 09:28:28,935 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da OK. 2023-11-12 09:28:28,936 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-12 09:28:28,936 main DEBUG OutputStream closed 2023-11-12 09:28:28,936 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-12 09:28:28,937 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-12 09:28:28,937 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@39529185 OK 2023-11-12 09:28:28,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-12 09:28:28,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-12 09:28:28,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-12 09:28:28,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-12 09:28:28,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-12 09:28:28,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-12 09:28:28,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-12 09:28:28,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-12 09:28:28,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-12 09:28:28,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-12 09:28:28,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-12 09:28:28,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-12 09:28:28,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-12 09:28:28,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-12 09:28:28,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-12 09:28:28,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-12 09:28:28,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-12 09:28:28,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-12 09:28:28,998 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12 09:28:28,998 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6a84a97d) with optional ClassLoader: null 2023-11-12 09:28:28,998 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-12 09:28:28,999 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@6a84a97d] started OK. 2023-11-12T09:28:29,033 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures timeout: 13 mins 2023-11-12 09:28:29,038 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-12 09:28:29,038 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12T09:28:29,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2 2023-11-12T09:28:29,386 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:28:29,389 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:29,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-12T09:28:29,395 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74, deleteOnExit=true 2023-11-12T09:28:29,395 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-12T09:28:29,396 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/test.cache.data in system properties and HBase conf 2023-11-12T09:28:29,396 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:28:29,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:28:29,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:28:29,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:28:29,398 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:28:29,525 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-12T09:28:30,001 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:28:30,006 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:28:30,006 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:28:30,006 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:28:30,007 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:28:30,007 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:28:30,007 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:28:30,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:28:30,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:28:30,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:28:30,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:28:30,009 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:28:30,009 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:28:30,009 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:28:30,009 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:28:30,451 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:28:30,456 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:28:30,748 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-12T09:28:30,886 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@41c133fb via org.mortbay.log.Slf4jLog 2023-11-12T09:28:30,907 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:28:30,962 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:28:31,006 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir/Jetty_localhost_38237_hdfs____15awxn/webapp 2023-11-12T09:28:31,133 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38237 2023-11-12T09:28:31,157 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:28:31,157 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:28:31,709 WARN [Listener at localhost/36805 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:28:31,777 WARN [Listener at localhost/36805 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:28:31,800 WARN [Listener at localhost/36805 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:28:31,806 INFO [Listener at localhost/36805 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:28:31,811 INFO [Listener at localhost/36805 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir/Jetty_localhost_32947_datanode____.b7lq70/webapp 2023-11-12T09:28:31,907 INFO [Listener at localhost/36805 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:32947 2023-11-12T09:28:32,178 WARN [Listener at localhost/33293 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:28:32,460 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:28:32,626 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe520d8df13186099: Processing first storage report for DS-3e2ede94-3d77-4401-8f33-81278e38f079 from datanode 4e120f22-d423-4d2b-aaab-dd185ce34744 2023-11-12T09:28:32,628 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe520d8df13186099: from storage DS-3e2ede94-3d77-4401-8f33-81278e38f079 node DatanodeRegistration(127.0.0.1:36527, datanodeUuid=4e120f22-d423-4d2b-aaab-dd185ce34744, infoPort=42829, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=1681812116;c=1699781310530), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-12T09:28:32,628 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe520d8df13186099: Processing first storage report for DS-5f52fb8d-5108-4660-b302-65e751789c09 from datanode 4e120f22-d423-4d2b-aaab-dd185ce34744 2023-11-12T09:28:32,628 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe520d8df13186099: from storage DS-5f52fb8d-5108-4660-b302-65e751789c09 node DatanodeRegistration(127.0.0.1:36527, datanodeUuid=4e120f22-d423-4d2b-aaab-dd185ce34744, infoPort=42829, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=1681812116;c=1699781310530), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-12T09:28:32,699 DEBUG [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2 2023-11-12T09:28:32,837 INFO [Listener at localhost/33293 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/zookeeper_0, clientPort=57937, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-12T09:28:32,857 INFO [Listener at localhost/33293 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57937 2023-11-12T09:28:32,864 INFO [Listener at localhost/33293 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:32,867 INFO [Listener at localhost/33293 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:33,530 INFO [Listener at localhost/33293 {}] util.FSUtils(462): Created version file at hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c with version=8 2023-11-12T09:28:33,530 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/hbase-staging 2023-11-12T09:28:33,547 DEBUG [Listener at localhost/33293 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-12T09:28:33,548 DEBUG [Listener at localhost/33293 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-12T09:28:33,548 DEBUG [Listener at localhost/33293 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-12T09:28:33,548 DEBUG [Listener at localhost/33293 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-12T09:28:33,761 DEBUG [Listener at localhost/33293 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-12T09:28:34,551 INFO [Listener at localhost/33293 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:28:34,599 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:28:34,600 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:28:34,600 INFO [Listener at localhost/33293 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:28:34,600 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:28:34,600 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:28:34,759 INFO [Listener at localhost/33293 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:28:34,887 INFO [Listener at localhost/33293 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-12T09:28:34,907 DEBUG [Listener at localhost/33293 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-12T09:28:34,913 INFO [Listener at localhost/33293 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:28:34,961 DEBUG [Listener at localhost/33293 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 20651 (auto-detected) 2023-11-12T09:28:34,962 DEBUG [Listener at localhost/33293 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:0f:ff:fe:94:aa:bf (auto-detected) 2023-11-12T09:28:35,000 INFO [Listener at localhost/33293 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:43745 2023-11-12T09:28:35,016 INFO [Listener at localhost/33293 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:35,018 INFO [Listener at localhost/33293 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:35,042 INFO [Listener at localhost/33293 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:43745 connecting to ZooKeeper ensemble=127.0.0.1:57937 2023-11-12T09:28:35,102 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:437450x0, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:28:35,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:43745-0x1004cf66f250000 connected 2023-11-12T09:28:35,142 DEBUG [Listener at localhost/33293 {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:28:35,146 DEBUG [Listener at localhost/33293 {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:28:35,150 DEBUG [Listener at localhost/33293 {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:28:35,165 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43745 2023-11-12T09:28:35,166 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43745 2023-11-12T09:28:35,166 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43745 2023-11-12T09:28:35,167 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43745 2023-11-12T09:28:35,167 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43745 2023-11-12T09:28:35,221 INFO [Listener at localhost/33293 {}] log.Log(170): Logging initialized @7074ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-12T09:28:35,341 INFO [Listener at localhost/33293 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:28:35,342 INFO [Listener at localhost/33293 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:28:35,342 INFO [Listener at localhost/33293 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:28:35,344 INFO [Listener at localhost/33293 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-12T09:28:35,345 INFO [Listener at localhost/33293 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:28:35,345 INFO [Listener at localhost/33293 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:28:35,348 INFO [Listener at localhost/33293 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:28:35,477 INFO [Listener at localhost/33293 {}] http.HttpServer(1227): Jetty bound to port 38987 2023-11-12T09:28:35,479 INFO [Listener at localhost/33293 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:28:35,512 INFO [Listener at localhost/33293 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:28:35,516 INFO [Listener at localhost/33293 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5c3f2af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:28:35,517 INFO [Listener at localhost/33293 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:28:35,517 INFO [Listener at localhost/33293 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6709d0da{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:28:35,584 INFO [Listener at localhost/33293 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:28:35,598 INFO [Listener at localhost/33293 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:28:35,598 INFO [Listener at localhost/33293 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:28:35,600 INFO [Listener at localhost/33293 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:28:35,607 INFO [Listener at localhost/33293 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:28:36,748 INFO [Listener at localhost/33293 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@387df152{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:28:36,760 INFO [Listener at localhost/33293 {}] server.AbstractConnector(333): Started ServerConnector@3cc8b64c{HTTP/1.1, (http/1.1)}{0.0.0.0:38987} 2023-11-12T09:28:36,760 INFO [Listener at localhost/33293 {}] server.Server(415): Started @8612ms 2023-11-12T09:28:36,764 INFO [Listener at localhost/33293 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c, hbase.cluster.distributed=false 2023-11-12T09:28:36,828 INFO [Listener at localhost/33293 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:28:36,829 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:28:36,829 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:28:36,829 INFO [Listener at localhost/33293 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:28:36,829 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:28:36,829 INFO [Listener at localhost/33293 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:28:36,838 INFO [Listener at localhost/33293 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:28:36,840 INFO [Listener at localhost/33293 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:28:36,841 INFO [Listener at localhost/33293 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37291 2023-11-12T09:28:36,844 INFO [Listener at localhost/33293 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-12T09:28:36,851 DEBUG [Listener at localhost/33293 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-12T09:28:36,852 INFO [Listener at localhost/33293 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:36,855 INFO [Listener at localhost/33293 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:36,857 INFO [Listener at localhost/33293 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:37291 connecting to ZooKeeper ensemble=127.0.0.1:57937 2023-11-12T09:28:36,862 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:372910x0, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:28:36,863 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:37291-0x1004cf66f250001 connected 2023-11-12T09:28:36,863 DEBUG [Listener at localhost/33293 {}] zookeeper.ZKUtil(113): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:28:36,864 DEBUG [Listener at localhost/33293 {}] zookeeper.ZKUtil(113): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:28:36,865 DEBUG [Listener at localhost/33293 {}] zookeeper.ZKUtil(113): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:28:36,866 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37291 2023-11-12T09:28:36,866 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37291 2023-11-12T09:28:36,867 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37291 2023-11-12T09:28:36,867 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37291 2023-11-12T09:28:36,867 DEBUG [Listener at localhost/33293 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37291 2023-11-12T09:28:36,871 INFO [Listener at localhost/33293 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:28:36,871 INFO [Listener at localhost/33293 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:28:36,872 INFO [Listener at localhost/33293 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:28:36,873 INFO [Listener at localhost/33293 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-12T09:28:36,873 INFO [Listener at localhost/33293 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:28:36,873 INFO [Listener at localhost/33293 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:28:36,873 INFO [Listener at localhost/33293 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:28:36,875 INFO [Listener at localhost/33293 {}] http.HttpServer(1227): Jetty bound to port 36931 2023-11-12T09:28:36,875 INFO [Listener at localhost/33293 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:28:36,881 INFO [Listener at localhost/33293 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:28:36,881 INFO [Listener at localhost/33293 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1650a57f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:28:36,882 INFO [Listener at localhost/33293 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:28:36,882 INFO [Listener at localhost/33293 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4c16fe40{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:28:36,899 INFO [Listener at localhost/33293 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:28:36,901 INFO [Listener at localhost/33293 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:28:36,901 INFO [Listener at localhost/33293 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:28:36,901 INFO [Listener at localhost/33293 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:28:36,903 INFO [Listener at localhost/33293 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:28:36,907 INFO [Listener at localhost/33293 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@462a200{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:28:36,909 INFO [Listener at localhost/33293 {}] server.AbstractConnector(333): Started ServerConnector@45201661{HTTP/1.1, (http/1.1)}{0.0.0.0:36931} 2023-11-12T09:28:36,909 INFO [Listener at localhost/33293 {}] server.Server(415): Started @8761ms 2023-11-12T09:28:36,914 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:28:36,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@481f5d5{HTTP/1.1, (http/1.1)}{0.0.0.0:40525} 2023-11-12T09:28:36,943 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8795ms 2023-11-12T09:28:36,943 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:36,962 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:28:36,962 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:28:36,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:36,966 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:43745 2023-11-12T09:28:36,985 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:28:36,985 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:28:36,985 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:36,985 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:36,986 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:28:36,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:28:36,988 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,43745,1699781313745 from backup master directory 2023-11-12T09:28:36,994 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:36,994 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:28:36,994 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:28:36,995 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:28:36,995 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:36,998 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-12T09:28:37,000 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-12T09:28:37,081 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/hbase.id with ID: 1a5153dc-a382-48d6-b2c4-28ad7e6ac1bc 2023-11-12T09:28:37,121 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:28:37,137 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:37,137 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:37,181 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:37,183 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-12T09:28:37,192 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-11-12T09:28:37,197 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-11-12T09:28:37,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-12T09:28:37,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-12T09:28:37,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:28:37,221 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:28:37,302 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store 2023-11-12T09:28:37,730 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-12T09:28:37,730 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:37,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:28:37,731 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:28:37,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:28:37,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:28:37,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:28:37,731 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:28:37,731 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:28:37,734 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/.initializing 2023-11-12T09:28:37,734 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/WALs/jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:37,736 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:28:37,745 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,43745,1699781313745.regiongroup-0 2023-11-12T09:28:37,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43745%2C1699781313745.jenkins-hbase3.apache.org%2C43745%2C1699781313745.regiongroup-0, suffix=, logDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/WALs/jenkins-hbase3.apache.org,43745,1699781313745, archiveDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/oldWALs, maxLogs=10 2023-11-12T09:28:37,795 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/WALs/jenkins-hbase3.apache.org,43745,1699781313745/jenkins-hbase3.apache.org%2C43745%2C1699781313745.jenkins-hbase3.apache.org%2C43745%2C1699781313745.regiongroup-0.1699781317766, exclude list is [], retry=0 2023-11-12T09:28:37,827 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:28:37,834 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:28:37,905 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/WALs/jenkins-hbase3.apache.org,43745,1699781313745/jenkins-hbase3.apache.org%2C43745%2C1699781313745.jenkins-hbase3.apache.org%2C43745%2C1699781313745.regiongroup-0.1699781317766 2023-11-12T09:28:37,906 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:28:37,906 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:37,907 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:37,911 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:37,912 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:37,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,014 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-12T09:28:38,020 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:38,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:38,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-12T09:28:38,026 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:38,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:38,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,029 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-12T09:28:38,029 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:38,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:38,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-12T09:28:38,033 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:38,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:38,038 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,039 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,055 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-12T09:28:38,058 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:28:38,062 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:38,063 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10610471040, jitterRate=-0.011822879314422607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-12T09:28:38,068 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:28:38,069 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-12T09:28:38,103 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4370c29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:38,158 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-12T09:28:38,178 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-12T09:28:38,178 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-12T09:28:38,181 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-12T09:28:38,183 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-12T09:28:38,189 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-11-12T09:28:38,189 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-12T09:28:38,224 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-12T09:28:38,239 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-12T09:28:38,242 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-12T09:28:38,245 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-12T09:28:38,246 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-12T09:28:38,248 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-12T09:28:38,250 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-12T09:28:38,257 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-12T09:28:38,260 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-12T09:28:38,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-12T09:28:38,262 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-12T09:28:38,277 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-12T09:28:38,279 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-12T09:28:38,284 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:28:38,284 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:28:38,284 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,284 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,285 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase3.apache.org,43745,1699781313745, sessionid=0x1004cf66f250000, setting cluster-up flag (Was=false) 2023-11-12T09:28:38,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-12T09:28:38,306 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,306 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,313 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-12T09:28:38,314 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:38,318 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,318 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,324 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-12T09:28:38,326 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:38,330 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,331 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:38,337 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-12T09:28:38,338 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:38,423 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:37291 2023-11-12T09:28:38,425 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(997): ClusterId : 1a5153dc-a382-48d6-b2c4-28ad7e6ac1bc 2023-11-12T09:28:38,427 INFO [RS:0;jenkins-hbase3:37291 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-12T09:28:38,431 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-12T09:28:38,437 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-12T09:28:38,437 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-12T09:28:38,441 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-12T09:28:38,441 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-12T09:28:38,444 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-12T09:28:38,445 DEBUG [RS:0;jenkins-hbase3:37291 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513d4ff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:38,447 DEBUG [RS:0;jenkins-hbase3:37291 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6cfb3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:28:38,452 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-12T09:28:38,452 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-12T09:28:38,452 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-12T09:28:38,456 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,43745,1699781313745 with isa=jenkins-hbase3.apache.org/172.31.12.81:37291, startcode=1699781316827 2023-11-12T09:28:38,480 DEBUG [RS:0;jenkins-hbase3:37291 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:28:38,482 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-12T09:28:38,495 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-12T09:28:38,502 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-12T09:28:38,517 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,43745,1699781313745 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-12T09:28:38,523 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:28:38,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:28:38,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:28:38,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:28:38,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-11-12T09:28:38,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,525 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:28:38,525 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,530 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1699781348530 2023-11-12T09:28:38,532 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-12T09:28:38,538 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-12T09:28:38,540 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:28:38,541 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-12T09:28:38,546 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48519, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:28:38,546 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:38,547 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:28:38,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-12T09:28:38,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-12T09:28:38,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-12T09:28:38,559 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-12T09:28:38,559 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-12T09:28:38,561 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43745 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:28:38,565 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,571 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-12T09:28:38,574 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-12T09:28:38,574 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-12T09:28:38,588 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5524d64b to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:28:38,597 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d219f04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:38,597 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:28:38,602 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-12T09:28:38,602 WARN [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-12T09:28:38,603 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-12T09:28:38,604 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-12T09:28:38,605 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781318605,5,FailOnTimeoutGroup] 2023-11-12T09:28:38,606 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781318605,5,FailOnTimeoutGroup] 2023-11-12T09:28:38,607 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,607 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-12T09:28:38,608 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,608 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,703 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,43745,1699781313745 with isa=jenkins-hbase3.apache.org/172.31.12.81:37291, startcode=1699781316827 2023-11-12T09:28:38,708 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43745 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:38,719 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:38,719 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:36805 2023-11-12T09:28:38,719 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38987 2023-11-12T09:28:38,729 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:28:38,729 DEBUG [RS:0;jenkins-hbase3:37291 {}] zookeeper.ZKUtil(111): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:38,729 WARN [RS:0;jenkins-hbase3:37291 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:28:38,730 INFO [RS:0;jenkins-hbase3:37291 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-11-12T09:28:38,730 INFO [RS:0;jenkins-hbase3:37291 {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-11-12T09:28:38,730 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:38,732 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,37291,1699781316827] 2023-11-12T09:28:38,747 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-12T09:28:38,767 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-12T09:28:38,785 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-12T09:28:38,791 INFO [RS:0;jenkins-hbase3:37291 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-12T09:28:38,791 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,793 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-12T09:28:38,802 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,803 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,803 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,803 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,803 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,803 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,803 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:28:38,804 DEBUG [RS:0;jenkins-hbase3:37291 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:28:38,805 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,806 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,806 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,806 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,806 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37291,1699781316827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:28:38,823 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-12T09:28:38,827 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37291,1699781316827-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:38,848 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,37291,1699781316827 started 2023-11-12T09:28:38,848 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,37291,1699781316827, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:37291, sessionid=0x1004cf66f250001 2023-11-12T09:28:38,849 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-12T09:28:38,849 DEBUG [RS:0;jenkins-hbase3:37291 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:38,849 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37291,1699781316827' 2023-11-12T09:28:38,849 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-12T09:28:38,850 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-12T09:28:38,850 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-12T09:28:38,850 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-12T09:28:38,851 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37291,1699781316827' 2023-11-12T09:28:38,851 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:38,851 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:38,852 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-12T09:28:38,852 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-12T09:28:38,852 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-12T09:28:38,852 DEBUG [RS:0;jenkins-hbase3:37291 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:38,852 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37291,1699781316827' 2023-11-12T09:28:38,852 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-12T09:28:38,853 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-12T09:28:38,853 DEBUG [RS:0;jenkins-hbase3:37291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-12T09:28:38,853 INFO [RS:0;jenkins-hbase3:37291 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-12T09:28:38,853 INFO [RS:0;jenkins-hbase3:37291 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-12T09:28:38,963 INFO [RS:0;jenkins-hbase3:37291 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:28:38,964 INFO [RS:0;jenkins-hbase3:37291 {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,37291,1699781316827.regiongroup-0 2023-11-12T09:28:38,967 INFO [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0, suffix=, logDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827, archiveDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs, maxLogs=32 2023-11-12T09:28:38,970 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-12T09:28:38,970 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:38,982 DEBUG [RS:0;jenkins-hbase3:37291 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781318968, exclude list is [], retry=0 2023-11-12T09:28:38,987 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:28:38,991 INFO [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781318968 2023-11-12T09:28:38,991 DEBUG [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:28:39,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:39,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:28:39,388 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:28:39,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:39,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:39,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:28:39,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:28:39,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:39,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:39,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:28:39,394 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:28:39,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:39,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:39,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740 2023-11-12T09:28:39,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740 2023-11-12T09:28:39,399 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:28:39,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:28:39,404 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:39,404 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11291316320, jitterRate=0.05158577859401703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:28:39,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:28:39,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:28:39,406 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:28:39,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:28:39,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:28:39,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:28:39,408 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:28:39,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:28:39,413 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:28:39,413 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-12T09:28:39,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-12T09:28:39,436 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-12T09:28:39,441 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-12T09:28:39,593 DEBUG [jenkins-hbase3:43745 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-12T09:28:39,602 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:39,608 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37291,1699781316827, state=OPENING 2023-11-12T09:28:39,618 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-12T09:28:39,619 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:39,619 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:39,620 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:28:39,620 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:28:39,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:39,770 INFO [AsyncFSWAL-0-hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 134 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:28:39,951 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:39,953 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-12T09:28:39,957 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-12T09:28:39,970 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-12T09:28:39,971 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-11-12T09:28:39,971 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-11-12T09:28:39,971 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:28:39,971 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-12T09:28:39,975 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta, suffix=.meta, logDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827, archiveDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs, maxLogs=32 2023-11-12T09:28:39,987 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta, exclude list is [], retry=0 2023-11-12T09:28:39,991 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:28:39,995 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta 2023-11-12T09:28:39,995 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:28:39,996 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:40,001 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:40,002 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-12T09:28:40,025 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-12T09:28:40,025 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-12T09:28:40,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-12T09:28:40,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:40,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-12T09:28:40,031 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-12T09:28:40,035 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:28:40,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:28:40,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:40,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:40,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:28:40,038 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:28:40,038 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:40,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:40,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:28:40,040 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:28:40,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:40,041 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:28:40,043 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740 2023-11-12T09:28:40,045 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740 2023-11-12T09:28:40,048 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:28:40,050 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:28:40,052 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11069592960, jitterRate=0.03093618154525757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:28:40,053 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:28:40,064 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1699781319942 2023-11-12T09:28:40,084 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-12T09:28:40,085 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-12T09:28:40,086 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:40,088 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37291,1699781316827, state=OPEN 2023-11-12T09:28:40,091 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:28:40,092 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:28:40,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:28:40,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:28:40,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-12T09:28:40,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37291,1699781316827 in 468 msec 2023-11-12T09:28:40,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-12T09:28:40,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 675 msec 2023-11-12T09:28:40,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7100 sec 2023-11-12T09:28:40,113 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1699781320113, completionTime=-1 2023-11-12T09:28:40,113 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-12T09:28:40,113 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-12T09:28:40,181 DEBUG [hconnection-0x2e7d6f30-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:40,184 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:40,208 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-12T09:28:40,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1699781380209 2023-11-12T09:28:40,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1699781440209 2023-11-12T09:28:40,209 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 95 msec 2023-11-12T09:28:40,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43745,1699781313745-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:40,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43745,1699781313745-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:40,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43745,1699781313745-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:40,236 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:43745, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:40,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-12T09:28:40,243 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-12T09:28:40,256 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-12T09:28:40,258 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:28:40,267 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-12T09:28:40,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:40,272 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:40,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:40,693 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9daf39e46084aad75149b683e2ae1693, NAME => 'hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:41,105 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:41,106 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 9daf39e46084aad75149b683e2ae1693, disabling compactions & flushes 2023-11-12T09:28:41,106 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,106 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,106 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. after waiting 0 ms 2023-11-12T09:28:41,106 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,106 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,106 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9daf39e46084aad75149b683e2ae1693: 2023-11-12T09:28:41,108 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:41,122 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1699781321110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781321110"}]},"ts":"1699781321110"} 2023-11-12T09:28:41,156 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:41,158 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:41,164 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781321158"}]},"ts":"1699781321158"} 2023-11-12T09:28:41,170 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-12T09:28:41,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=9daf39e46084aad75149b683e2ae1693, ASSIGN}] 2023-11-12T09:28:41,178 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=9daf39e46084aad75149b683e2ae1693, ASSIGN 2023-11-12T09:28:41,180 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=9daf39e46084aad75149b683e2ae1693, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:41,331 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=9daf39e46084aad75149b683e2ae1693, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:41,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 9daf39e46084aad75149b683e2ae1693, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:41,489 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:41,495 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,495 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:28:41,496 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,37291,1699781316827.regiongroup-1 2023-11-12T09:28:41,498 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1, suffix=, logDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827, archiveDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs, maxLogs=32 2023-11-12T09:28:41,511 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781321500, exclude list is [], retry=0 2023-11-12T09:28:41,515 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:28:41,519 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781321500 2023-11-12T09:28:41,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:28:41,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 9daf39e46084aad75149b683e2ae1693, NAME => 'hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:41,520 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:41,521 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,521 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:41,521 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,521 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,523 INFO [StoreOpener-9daf39e46084aad75149b683e2ae1693-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,525 INFO [StoreOpener-9daf39e46084aad75149b683e2ae1693-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9daf39e46084aad75149b683e2ae1693 columnFamilyName info 2023-11-12T09:28:41,525 DEBUG [StoreOpener-9daf39e46084aad75149b683e2ae1693-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:41,525 INFO [StoreOpener-9daf39e46084aad75149b683e2ae1693-1 {}] regionserver.HStore(324): Store=9daf39e46084aad75149b683e2ae1693/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:41,527 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,528 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,530 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:28:41,534 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:41,535 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 9daf39e46084aad75149b683e2ae1693; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11131514080, jitterRate=0.03670303523540497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:41,536 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 9daf39e46084aad75149b683e2ae1693: 2023-11-12T09:28:41,537 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693., pid=6, masterSystemTime=1699781321489 2023-11-12T09:28:41,541 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,541 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:28:41,542 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=9daf39e46084aad75149b683e2ae1693, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:41,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-12T09:28:41,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 9daf39e46084aad75149b683e2ae1693, server=jenkins-hbase3.apache.org,37291,1699781316827 in 210 msec 2023-11-12T09:28:41,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-12T09:28:41,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=9daf39e46084aad75149b683e2ae1693, ASSIGN in 372 msec 2023-11-12T09:28:41,553 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:41,554 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781321554"}]},"ts":"1699781321554"} 2023-11-12T09:28:41,556 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-12T09:28:41,560 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:41,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3010 sec 2023-11-12T09:28:41,571 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-12T09:28:41,572 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:41,572 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:41,573 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:28:41,631 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-12T09:28:41,646 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:41,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 33 msec 2023-11-12T09:28:41,655 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-12T09:28:41,667 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:41,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 16 msec 2023-11-12T09:28:41,681 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-12T09:28:41,685 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-12T09:28:41,685 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 4.690sec 2023-11-12T09:28:41,687 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-12T09:28:41,689 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-12T09:28:41,690 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-12T09:28:41,691 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-12T09:28:41,691 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-12T09:28:41,692 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43745,1699781313745-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:28:41,693 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43745,1699781313745-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-12T09:28:41,706 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-12T09:28:41,707 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-12T09:28:41,778 DEBUG [Listener at localhost/33293 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x51a691c9 to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:28:41,782 DEBUG [Listener at localhost/33293 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbcef53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:41,787 DEBUG [Listener at localhost/33293 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-12T09:28:41,787 DEBUG [Listener at localhost/33293 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-12T09:28:41,801 DEBUG [hconnection-0x45030a3a-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:41,812 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46058, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:41,823 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:28:41,823 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-12T09:28:41,823 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/test.cache.data in system properties and HBase conf 2023-11-12T09:28:41,823 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:28:41,824 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:28:41,825 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:28:41,825 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:28:41,825 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:28:41,825 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:28:41,825 INFO [Listener at localhost/33293 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-12T09:28:43,305 WARN [Thread-251 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:28:43,691 INFO [Thread-251 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:28:43,698 INFO [Thread-251 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43399_jobhistory____.ve9lhe/webapp 2023-11-12T09:28:43,866 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-12T09:28:43,866 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-12T09:28:43,867 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:28:43,870 INFO [Thread-251 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:28:43,968 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:44,096 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:44,371 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-12T09:28:44,407 INFO [Thread-251 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43399 2023-11-12T09:28:44,758 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-12T09:28:44,759 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-12T09:28:44,801 WARN [Listener at jenkins-hbase3.apache.org/34997 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:28:44,883 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-12T09:28:44,883 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:44,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-12T09:28:44,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:44,896 WARN [Listener at jenkins-hbase3.apache.org/34997 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:28:44,896 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:28:44,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:44,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-12T09:28:44,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:28:44,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-12T09:28:44,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-12T09:28:44,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:44,903 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_35133_cluster____xumfo/webapp 2023-11-12T09:28:44,975 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:28:44,975 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-12T09:28:44,975 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:28:44,976 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:28:44,986 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:45,219 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:45,359 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:45,366 INFO [Listener at jenkins-hbase3.apache.org/34997 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35133 2023-11-12T09:28:45,574 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:45,804 WARN [Listener at jenkins-hbase3.apache.org/41385 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:28:45,828 WARN [Listener at jenkins-hbase3.apache.org/41385 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:28:45,828 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:28:45,836 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46325_node____r4bqc3/webapp 2023-11-12T09:28:45,889 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:28:45,889 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:28:45,889 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:28:45,889 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:28:45,896 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:45,918 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:45,986 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:45,996 INFO [Listener at jenkins-hbase3.apache.org/41385 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46325 2023-11-12T09:28:46,132 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:28:46,141 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:28:46,143 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:28:46,153 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45269_node____.6tzybx/webapp 2023-11-12T09:28:46,209 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:28:46,210 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:28:46,210 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:28:46,210 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:28:46,217 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:46,233 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:46,296 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:28:46,303 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45269 2023-11-12T09:28:46,335 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-12T09:28:46,335 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:36805/backupUT 2023-11-12T09:28:46,346 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:46,350 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35508, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:46,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-11-12T09:28:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-12T09:28:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:28:46,389 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:46,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 33 msec 2023-11-12T09:28:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:28:46,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-11-12T09:28:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-12T09:28:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:28:46,494 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:46,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 19 msec 2023-11-12T09:28:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:28:46,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-11-12T09:28:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-12T09:28:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:28:46,603 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:46,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 15 msec 2023-11-12T09:28:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:28:46,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-11-12T09:28:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-12T09:28:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:28:46,714 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:46,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-11-12T09:28:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:28:46,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'test-1699781326335', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1699781326335 2023-11-12T09:28:46,827 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1699781326335 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:46,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1699781326335" procId is: 13 2023-11-12T09:28:46,828 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:46,829 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1699781326335 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:28:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:28:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:28:47,261 INFO [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 355fc0ccb227f73de53d754779d91940, NAME => 'test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699781326335', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:47,287 DEBUG [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:47,287 DEBUG [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1668): Closing 355fc0ccb227f73de53d754779d91940, disabling compactions & flushes 2023-11-12T09:28:47,288 INFO [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,288 DEBUG [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,288 DEBUG [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. after waiting 0 ms 2023-11-12T09:28:47,288 DEBUG [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,288 INFO [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1909): Closed test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,288 DEBUG [RegionOpenAndInit-test-1699781326335-pool-0 {}] regionserver.HRegion(1622): Region close journal for 355fc0ccb227f73de53d754779d91940: 2023-11-12T09:28:47,289 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1699781326335 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:47,290 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699781327290"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781327290"}]},"ts":"1699781327290"} 2023-11-12T09:28:47,293 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:47,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1699781326335 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:47,294 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781326335","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781327294"}]},"ts":"1699781327294"} 2023-11-12T09:28:47,297 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781326335, state=ENABLING in hbase:meta 2023-11-12T09:28:47,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, ASSIGN}] 2023-11-12T09:28:47,305 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, ASSIGN 2023-11-12T09:28:47,306 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:28:47,458 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=355fc0ccb227f73de53d754779d91940, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:47,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 355fc0ccb227f73de53d754779d91940, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:47,614 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:47,622 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,622 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 355fc0ccb227f73de53d754779d91940, NAME => 'test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:47,623 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:47,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781326335 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:47,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,625 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,628 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 355fc0ccb227f73de53d754779d91940 columnFamilyName f 2023-11-12T09:28:47,628 DEBUG [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:47,629 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.HStore(324): Store=355fc0ccb227f73de53d754779d91940/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:47,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,631 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,636 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:47,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:47,640 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 355fc0ccb227f73de53d754779d91940; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12050335520, jitterRate=0.12227495014667511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:47,641 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 355fc0ccb227f73de53d754779d91940: 2023-11-12T09:28:47,643 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940., pid=15, masterSystemTime=1699781327613 2023-11-12T09:28:47,645 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,645 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:47,646 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=355fc0ccb227f73de53d754779d91940, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:47,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-12T09:28:47,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 355fc0ccb227f73de53d754779d91940, server=jenkins-hbase3.apache.org,37291,1699781316827 in 189 msec 2023-11-12T09:28:47,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-12T09:28:47,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, ASSIGN in 350 msec 2023-11-12T09:28:47,655 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1699781326335 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:47,655 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781326335","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781327655"}]},"ts":"1699781327655"} 2023-11-12T09:28:47,657 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781326335, state=ENABLED in hbase:meta 2023-11-12T09:28:47,660 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1699781326335 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:47,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1699781326335 in 837 msec 2023-11-12T09:28:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:28:47,942 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1699781326335, procId: 13 completed 2023-11-12T09:28:47,944 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x19982eae to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:28:47,949 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732899a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:47,952 DEBUG [hconnection-0x42d775fa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:47,954 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44730, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:47,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37291 {}] regionserver.HRegion(8210): writing data to region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:28:48,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:test-16997813263351', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16997813263351 2023-11-12T09:28:48,059 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997813263351 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:48,059 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:48,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16997813263351" procId is: 16 2023-11-12T09:28:48,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16997813263351 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:28:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:28:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:28:48,477 INFO [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7a46314c3b11bd429216bed3cf7244cb, NAME => 'ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997813263351', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:28:48,891 DEBUG [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:48,891 DEBUG [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1668): Closing 7a46314c3b11bd429216bed3cf7244cb, disabling compactions & flushes 2023-11-12T09:28:48,891 INFO [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:48,891 DEBUG [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:48,891 DEBUG [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. after waiting 0 ms 2023-11-12T09:28:48,891 DEBUG [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:48,891 INFO [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:48,891 DEBUG [RegionOpenAndInit-ns2:test-16997813263351-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7a46314c3b11bd429216bed3cf7244cb: 2023-11-12T09:28:48,893 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16997813263351 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:48,893 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781328893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781328893"}]},"ts":"1699781328893"} 2023-11-12T09:28:48,896 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:48,897 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16997813263351 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:48,897 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997813263351","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781328897"}]},"ts":"1699781328897"} 2023-11-12T09:28:48,899 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997813263351, state=ENABLING in hbase:meta 2023-11-12T09:28:48,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997813263351, region=7a46314c3b11bd429216bed3cf7244cb, ASSIGN}] 2023-11-12T09:28:48,908 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997813263351, region=7a46314c3b11bd429216bed3cf7244cb, ASSIGN 2023-11-12T09:28:48,909 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997813263351, region=7a46314c3b11bd429216bed3cf7244cb, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:49,061 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=7a46314c3b11bd429216bed3cf7244cb, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:49,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 7a46314c3b11bd429216bed3cf7244cb, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:28:49,215 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:49,219 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:49,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 7a46314c3b11bd429216bed3cf7244cb, NAME => 'ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:49,220 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:49,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997813263351 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:49,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,222 INFO [StoreOpener-7a46314c3b11bd429216bed3cf7244cb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,224 INFO [StoreOpener-7a46314c3b11bd429216bed3cf7244cb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a46314c3b11bd429216bed3cf7244cb columnFamilyName f 2023-11-12T09:28:49,224 DEBUG [StoreOpener-7a46314c3b11bd429216bed3cf7244cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:49,225 INFO [StoreOpener-7a46314c3b11bd429216bed3cf7244cb-1 {}] regionserver.HStore(324): Store=7a46314c3b11bd429216bed3cf7244cb/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:49,226 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,227 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,230 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:28:49,237 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:49,238 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 7a46314c3b11bd429216bed3cf7244cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10738976960, jitterRate=1.4516711235046387E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:49,239 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 7a46314c3b11bd429216bed3cf7244cb: 2023-11-12T09:28:49,240 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb., pid=18, masterSystemTime=1699781329215 2023-11-12T09:28:49,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:49,243 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:28:49,244 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=7a46314c3b11bd429216bed3cf7244cb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:49,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-12T09:28:49,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 7a46314c3b11bd429216bed3cf7244cb, server=jenkins-hbase3.apache.org,37291,1699781316827 in 183 msec 2023-11-12T09:28:49,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-12T09:28:49,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997813263351, region=7a46314c3b11bd429216bed3cf7244cb, ASSIGN in 343 msec 2023-11-12T09:28:49,253 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16997813263351 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:49,253 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997813263351","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781329253"}]},"ts":"1699781329253"} 2023-11-12T09:28:49,255 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997813263351, state=ENABLED in hbase:meta 2023-11-12T09:28:49,259 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997813263351 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:49,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16997813263351 in 1.2030 sec 2023-11-12T09:28:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:28:50,169 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-16997813263351, procId: 16 completed 2023-11-12T09:28:50,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37291 {}] regionserver.HRegion(8210): writing data to region ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:28:50,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:test-16997813263352', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16997813263352 2023-11-12T09:28:50,260 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997813263352 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:50,261 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:50,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16997813263352" procId is: 19 2023-11-12T09:28:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:28:50,262 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16997813263352 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:50,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:28:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:28:50,681 INFO [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4330779e284cca1387db0585407c975a, NAME => 'ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997813263352', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:50,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16997813263351' 2023-11-12T09:28:50,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1699781326335' 2023-11-12T09:28:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:28:51,097 DEBUG [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:51,097 DEBUG [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1668): Closing 4330779e284cca1387db0585407c975a, disabling compactions & flushes 2023-11-12T09:28:51,097 INFO [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,097 DEBUG [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,097 DEBUG [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. after waiting 0 ms 2023-11-12T09:28:51,097 DEBUG [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,097 INFO [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,097 DEBUG [RegionOpenAndInit-ns3:test-16997813263352-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4330779e284cca1387db0585407c975a: 2023-11-12T09:28:51,099 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16997813263352 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:51,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781331099"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781331099"}]},"ts":"1699781331099"} 2023-11-12T09:28:51,101 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:51,102 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16997813263352 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:51,102 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997813263352","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781331102"}]},"ts":"1699781331102"} 2023-11-12T09:28:51,104 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997813263352, state=ENABLING in hbase:meta 2023-11-12T09:28:51,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997813263352, region=4330779e284cca1387db0585407c975a, ASSIGN}] 2023-11-12T09:28:51,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997813263352, region=4330779e284cca1387db0585407c975a, ASSIGN 2023-11-12T09:28:51,119 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997813263352, region=4330779e284cca1387db0585407c975a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:51,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=4330779e284cca1387db0585407c975a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:51,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 4330779e284cca1387db0585407c975a, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:28:51,425 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:51,431 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 4330779e284cca1387db0585407c975a, NAME => 'ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:51,431 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:51,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997813263352 4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:51,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,434 INFO [StoreOpener-4330779e284cca1387db0585407c975a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,435 INFO [StoreOpener-4330779e284cca1387db0585407c975a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4330779e284cca1387db0585407c975a columnFamilyName f 2023-11-12T09:28:51,436 DEBUG [StoreOpener-4330779e284cca1387db0585407c975a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:51,436 INFO [StoreOpener-4330779e284cca1387db0585407c975a-1 {}] regionserver.HStore(324): Store=4330779e284cca1387db0585407c975a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:51,437 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns3/test-16997813263352/4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,438 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns3/test-16997813263352/4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,442 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 4330779e284cca1387db0585407c975a 2023-11-12T09:28:51,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns3/test-16997813263352/4330779e284cca1387db0585407c975a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:51,445 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 4330779e284cca1387db0585407c975a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10322041920, jitterRate=-0.03868493437767029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:51,446 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 4330779e284cca1387db0585407c975a: 2023-11-12T09:28:51,447 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a., pid=21, masterSystemTime=1699781331425 2023-11-12T09:28:51,450 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,450 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:28:51,451 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=4330779e284cca1387db0585407c975a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:51,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-12T09:28:51,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 4330779e284cca1387db0585407c975a, server=jenkins-hbase3.apache.org,37291,1699781316827 in 180 msec 2023-11-12T09:28:51,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-12T09:28:51,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997813263352, region=4330779e284cca1387db0585407c975a, ASSIGN in 339 msec 2023-11-12T09:28:51,459 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16997813263352 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:51,459 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997813263352","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781331459"}]},"ts":"1699781331459"} 2023-11-12T09:28:51,461 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997813263352, state=ENABLED in hbase:meta 2023-11-12T09:28:51,465 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997813263352 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:51,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16997813263352 in 1.2090 sec 2023-11-12T09:28:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:28:52,371 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-16997813263352, procId: 19 completed 2023-11-12T09:28:52,372 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16997813263352 get assigned. Timeout = 60000ms 2023-11-12T09:28:52,374 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:28:52,378 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16997813263352 assigned to meta. Checking AM states. 2023-11-12T09:28:52,379 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:28:52,379 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16997813263352 assigned. 2023-11-12T09:28:52,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns4:test-16997813263353', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16997813263353 2023-11-12T09:28:52,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997813263353 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:52,385 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:52,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16997813263353" procId is: 22 2023-11-12T09:28:52,386 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16997813263353 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:28:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:28:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:28:52,799 INFO [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 6ab12e4c0c19d984646cbe9c7422a1e3, NAME => 'ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16997813263353', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:28:53,132 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:53,213 DEBUG [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:53,213 DEBUG [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1668): Closing 6ab12e4c0c19d984646cbe9c7422a1e3, disabling compactions & flushes 2023-11-12T09:28:53,213 INFO [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,214 DEBUG [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,214 DEBUG [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. after waiting 0 ms 2023-11-12T09:28:53,214 DEBUG [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,214 INFO [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,214 DEBUG [RegionOpenAndInit-ns4:test-16997813263353-pool-0 {}] regionserver.HRegion(1622): Region close journal for 6ab12e4c0c19d984646cbe9c7422a1e3: 2023-11-12T09:28:53,215 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16997813263353 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:53,216 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781333215"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781333215"}]},"ts":"1699781333215"} 2023-11-12T09:28:53,219 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:53,220 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16997813263353 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:53,220 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997813263353","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781333220"}]},"ts":"1699781333220"} 2023-11-12T09:28:53,222 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997813263353, state=ENABLING in hbase:meta 2023-11-12T09:28:53,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997813263353, region=6ab12e4c0c19d984646cbe9c7422a1e3, ASSIGN}] 2023-11-12T09:28:53,228 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997813263353, region=6ab12e4c0c19d984646cbe9c7422a1e3, ASSIGN 2023-11-12T09:28:53,229 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16997813263353, region=6ab12e4c0c19d984646cbe9c7422a1e3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:53,381 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=6ab12e4c0c19d984646cbe9c7422a1e3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:53,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 6ab12e4c0c19d984646cbe9c7422a1e3, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:28:53,536 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:53,540 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 6ab12e4c0c19d984646cbe9c7422a1e3, NAME => 'ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:53,541 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:53,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997813263353 6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:53,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,543 INFO [StoreOpener-6ab12e4c0c19d984646cbe9c7422a1e3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,544 INFO [StoreOpener-6ab12e4c0c19d984646cbe9c7422a1e3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ab12e4c0c19d984646cbe9c7422a1e3 columnFamilyName f 2023-11-12T09:28:53,545 DEBUG [StoreOpener-6ab12e4c0c19d984646cbe9c7422a1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:53,545 INFO [StoreOpener-6ab12e4c0c19d984646cbe9c7422a1e3-1 {}] regionserver.HStore(324): Store=6ab12e4c0c19d984646cbe9c7422a1e3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:53,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns4/test-16997813263353/6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns4/test-16997813263353/6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:28:53,552 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns4/test-16997813263353/6ab12e4c0c19d984646cbe9c7422a1e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:53,553 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 6ab12e4c0c19d984646cbe9c7422a1e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10707241760, jitterRate=-0.0028104037046432495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:53,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 6ab12e4c0c19d984646cbe9c7422a1e3: 2023-11-12T09:28:53,555 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3., pid=24, masterSystemTime=1699781333535 2023-11-12T09:28:53,557 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,557 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:28:53,558 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=6ab12e4c0c19d984646cbe9c7422a1e3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:53,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-12T09:28:53,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 6ab12e4c0c19d984646cbe9c7422a1e3, server=jenkins-hbase3.apache.org,37291,1699781316827 in 176 msec 2023-11-12T09:28:53,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-12T09:28:53,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16997813263353, region=6ab12e4c0c19d984646cbe9c7422a1e3, ASSIGN in 336 msec 2023-11-12T09:28:53,565 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16997813263353 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:53,565 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997813263353","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781333565"}]},"ts":"1699781333565"} 2023-11-12T09:28:53,567 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997813263353, state=ENABLED in hbase:meta 2023-11-12T09:28:53,569 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997813263353 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:53,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16997813263353 in 1.1890 sec 2023-11-12T09:28:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:28:54,495 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-16997813263353, procId: 22 completed 2023-11-12T09:28:54,495 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16997813263353 get assigned. Timeout = 60000ms 2023-11-12T09:28:54,495 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:28:54,499 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16997813263353 assigned to meta. Checking AM states. 2023-11-12T09:28:54,500 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:28:54,500 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16997813263353 assigned. 2023-11-12T09:28:54,500 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x19982eae to 127.0.0.1:57937 2023-11-12T09:28:54,500 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:54,540 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=587, OpenFileDescriptor=696, MaxFileDescriptor=60000, SystemLoadAverage=193, ProcessCount=171, AvailableMemoryMB=5378 2023-11-12T09:28:54,540 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.ResourceChecker(130): Thread=587 is superior to 500 2023-11-12T09:28:54,543 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(82): create full backup image for all tables 2023-11-12T09:28:54,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$14(2731): Client=jenkins//172.31.12.81 modify table test-1699781326335 from 'test-1699781326335', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1699781326335', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=test-1699781326335 2023-11-12T09:28:54,882 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997813263351 2023-11-12T09:28:54,882 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997813263351 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:54,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997813263352 2023-11-12T09:28:54,884 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997813263352 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:54,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997813263353 2023-11-12T09:28:54,884 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997813263353 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:54,885 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699781326335 2023-11-12T09:28:54,885 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1699781326335 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:54,987 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/.tabledesc/.tableinfo.0000000001.367 2023-11-12T09:28:54,987 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/.tabledesc/.tableinfo.0000000002.598 2023-11-12T09:28:54,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=test-1699781326335}] 2023-11-12T09:28:55,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, REOPEN/MOVE}] 2023-11-12T09:28:55,009 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, REOPEN/MOVE 2023-11-12T09:28:55,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=355fc0ccb227f73de53d754779d91940, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:55,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; CloseRegionProcedure 355fc0ccb227f73de53d754779d91940, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:55,170 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:55,172 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(119): Close 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:55,174 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1668): Closing 355fc0ccb227f73de53d754779d91940, disabling compactions & flushes 2023-11-12T09:28:55,174 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1690): Closing region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:55,174 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:55,174 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1778): Acquired close lock on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. after waiting 0 ms 2023-11-12T09:28:55,174 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1788): Updates disabled for region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:55,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2819): Flushing 355fc0ccb227f73de53d754779d91940 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:28:55,256 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f/1f211f4489a24367a0d18ad65d845250 is 37, key is row10/f:q1/1699781327970/Put/seqid=0 2023-11-12T09:28:55,676 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f/1f211f4489a24367a0d18ad65d845250 2023-11-12T09:28:55,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f/1f211f4489a24367a0d18ad65d845250 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250 2023-11-12T09:28:55,772 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:28:55,776 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 355fc0ccb227f73de53d754779d91940 in 602ms, sequenceid=103, compaction requested=false 2023-11-12T09:28:55,785 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-12T09:28:55,786 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:55,787 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1909): Closed test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:55,787 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1622): Region close journal for 355fc0ccb227f73de53d754779d91940: 2023-11-12T09:28:55,788 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionServer(3735): Not adding moved region record: 355fc0ccb227f73de53d754779d91940 to self. 2023-11-12T09:28:55,790 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(163): Closed 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:55,791 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=355fc0ccb227f73de53d754779d91940, regionState=CLOSED 2023-11-12T09:28:55,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-12T09:28:55,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; CloseRegionProcedure 355fc0ccb227f73de53d754779d91940, server=jenkins-hbase3.apache.org,37291,1699781316827 in 780 msec 2023-11-12T09:28:55,796 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=true 2023-11-12T09:28:55,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=355fc0ccb227f73de53d754779d91940, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:55,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=27, state=RUNNABLE; OpenRegionProcedure 355fc0ccb227f73de53d754779d91940, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:56,101 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:56,106 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:56,106 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 355fc0ccb227f73de53d754779d91940, NAME => 'test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:56,107 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:56,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781326335 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:56,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,109 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,111 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 355fc0ccb227f73de53d754779d91940 columnFamilyName f 2023-11-12T09:28:56,111 DEBUG [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:56,122 DEBUG [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250 2023-11-12T09:28:56,122 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.HStore(324): Store=355fc0ccb227f73de53d754779d91940/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:56,122 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,124 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 355fc0ccb227f73de53d754779d91940 columnFamilyName f3 2023-11-12T09:28:56,124 DEBUG [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:56,124 INFO [StoreOpener-355fc0ccb227f73de53d754779d91940-1 {}] regionserver.HStore(324): Store=355fc0ccb227f73de53d754779d91940/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:56,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,129 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1699781326335 descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:28:56,131 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:28:56,132 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 355fc0ccb227f73de53d754779d91940; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11811070720, jitterRate=0.09999167919158936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:28:56,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 355fc0ccb227f73de53d754779d91940: 2023-11-12T09:28:56,134 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940., pid=29, masterSystemTime=1699781336101 2023-11-12T09:28:56,136 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:56,136 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:28:56,137 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=355fc0ccb227f73de53d754779d91940, regionState=OPEN, openSeqNum=107, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:56,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=27 2023-11-12T09:28:56,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=27, state=SUCCESS; OpenRegionProcedure 355fc0ccb227f73de53d754779d91940, server=jenkins-hbase3.apache.org,37291,1699781316827 in 190 msec 2023-11-12T09:28:56,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-12T09:28:56,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=test-1699781326335, region=355fc0ccb227f73de53d754779d91940, REOPEN/MOVE in 1.1340 sec 2023-11-12T09:28:56,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=26, resume processing ppid=25 2023-11-12T09:28:56,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, ppid=25, state=SUCCESS; ReopenTableRegionsProcedure table=test-1699781326335 in 1.1550 sec 2023-11-12T09:28:56,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; ModifyTableProcedure table=test-1699781326335 in 1.5950 sec 2023-11-12T09:28:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:28:56,162 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x75a4911b to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:28:56,167 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60cc93a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:56,169 DEBUG [hconnection-0x5e629fdc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:56,171 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46510, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:56,205 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:56,206 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:50590, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:56,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-11-12T09:28:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-12T09:28:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2023-11-12T09:28:56,225 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:28:56,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 13 msec 2023-11-12T09:28:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2023-11-12T09:28:56,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-12T09:28:56,343 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:56,343 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:56,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 31 2023-11-12T09:28:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-12T09:28:56,345 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-12T09:28:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-12T09:28:56,763 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997813263352' 2023-11-12T09:28:56,764 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16997813263353' 2023-11-12T09:28:56,768 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-12T09:28:57,181 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:57,181 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 2bd6ff45da8970c8169b00517ffa7e18, disabling compactions & flushes 2023-11-12T09:28:57,181 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,181 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,181 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. after waiting 0 ms 2023-11-12T09:28:57,181 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,181 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,181 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:28:57,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:57,183 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781337183"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781337183"}]},"ts":"1699781337183"} 2023-11-12T09:28:57,185 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:57,186 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:57,186 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781337186"}]},"ts":"1699781337186"} 2023-11-12T09:28:57,188 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:28:57,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN}] 2023-11-12T09:28:57,195 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN 2023-11-12T09:28:57,196 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:57,347 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:57,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-12T09:28:57,502 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:57,509 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(130): Open backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,509 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7254): Opening region: {ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:57,510 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:57,510 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,510 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(888): Instantiated backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:57,510 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7296): checking encryption for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,510 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7299): checking classloading for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,512 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,514 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName meta 2023-11-12T09:28:57,514 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:57,515 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:57,515 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,516 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName session 2023-11-12T09:28:57,517 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:57,517 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:57,519 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,519 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,522 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:28:57,523 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1076): writing seq id for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:28:57,526 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:57,527 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): Opened 2bd6ff45da8970c8169b00517ffa7e18; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9931745440, jitterRate=-0.07503412663936615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:28:57,527 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(992): Region open journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:28:57,528 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., pid=33, masterSystemTime=1699781337502 2023-11-12T09:28:57,530 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,530 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(158): Opened backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:28:57,531 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:57,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-12T09:28:57,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 183 msec 2023-11-12T09:28:57,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=32, resume processing ppid=31 2023-11-12T09:28:57,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, ppid=31, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN in 342 msec 2023-11-12T09:28:57,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:57,539 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781337539"}]},"ts":"1699781337539"} 2023-11-12T09:28:57,540 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:28:57,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:57,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, state=SUCCESS; CreateTableProcedure table=backup:system in 1.2060 sec 2023-11-12T09:28:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-12T09:28:58,454 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 31 completed 2023-11-12T09:28:58,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-12T09:28:58,471 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:58,472 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:58,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 34 2023-11-12T09:28:58,473 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:28:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:28:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:28:58,890 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3c0632ff163ee5a4367c764a083d82ca, NAME => 'backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:28:59,044 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:28:59,303 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:59,303 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 3c0632ff163ee5a4367c764a083d82ca, disabling compactions & flushes 2023-11-12T09:28:59,303 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,303 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,303 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. after waiting 0 ms 2023-11-12T09:28:59,303 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,303 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,303 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3c0632ff163ee5a4367c764a083d82ca: 2023-11-12T09:28:59,305 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:59,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699781339305"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781339305"}]},"ts":"1699781339305"} 2023-11-12T09:28:59,307 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:59,308 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:59,308 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781339308"}]},"ts":"1699781339308"} 2023-11-12T09:28:59,310 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-12T09:28:59,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=3c0632ff163ee5a4367c764a083d82ca, ASSIGN}] 2023-11-12T09:28:59,318 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=3c0632ff163ee5a4367c764a083d82ca, ASSIGN 2023-11-12T09:28:59,319 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=3c0632ff163ee5a4367c764a083d82ca, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37291,1699781316827; forceNewPlan=false, retain=false 2023-11-12T09:28:59,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=3c0632ff163ee5a4367c764a083d82ca, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:59,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure 3c0632ff163ee5a4367c764a083d82ca, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:28:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:28:59,625 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:59,629 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7254): Opening region: {ENCODED => 3c0632ff163ee5a4367c764a083d82ca, NAME => 'backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:59,630 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:59,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:59,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7296): checking encryption for 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,630 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7299): checking classloading for 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,632 INFO [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,633 INFO [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c0632ff163ee5a4367c764a083d82ca columnFamilyName meta 2023-11-12T09:28:59,633 DEBUG [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:59,634 INFO [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] regionserver.HStore(324): Store=3c0632ff163ee5a4367c764a083d82ca/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:59,634 INFO [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,636 INFO [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c0632ff163ee5a4367c764a083d82ca columnFamilyName session 2023-11-12T09:28:59,636 DEBUG [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:59,636 INFO [StoreOpener-3c0632ff163ee5a4367c764a083d82ca-1 {}] regionserver.HStore(324): Store=3c0632ff163ee5a4367c764a083d82ca/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:59,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system_bulk/3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,638 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system_bulk/3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,639 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:28:59,641 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1076): writing seq id for 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:28:59,643 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system_bulk/3c0632ff163ee5a4367c764a083d82ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:59,644 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): Opened 3c0632ff163ee5a4367c764a083d82ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9451354240, jitterRate=-0.11977404356002808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:28:59,645 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(992): Region open journal for 3c0632ff163ee5a4367c764a083d82ca: 2023-11-12T09:28:59,645 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca., pid=36, masterSystemTime=1699781339624 2023-11-12T09:28:59,647 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,647 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:28:59,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=3c0632ff163ee5a4367c764a083d82ca, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:28:59,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-11-12T09:28:59,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure 3c0632ff163ee5a4367c764a083d82ca, server=jenkins-hbase3.apache.org,37291,1699781316827 in 178 msec 2023-11-12T09:28:59,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-12T09:28:59,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=3c0632ff163ee5a4367c764a083d82ca, ASSIGN in 337 msec 2023-11-12T09:28:59,656 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:59,656 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781339656"}]},"ts":"1699781339656"} 2023-11-12T09:28:59,658 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-12T09:28:59,661 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:59,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1930 sec 2023-11-12T09:29:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:29:00,581 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 34 completed 2023-11-12T09:29:00,591 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37291 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-12T09:29:00,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781340628 (current time:1699781340628). 2023-11-12T09:29:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x66db17c2 to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:29:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6ebf19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:00,641 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46514, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x66db17c2 to 127.0.0.1:57937 2023-11-12T09:29:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-11-12T09:29:00,658 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-12T09:29:00,664 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:00,676 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:00,678 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-12T09:29:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-12T09:29:01,091 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:01,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18}] 2023-11-12T09:29:01,102 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:01,260 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:01,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=38 2023-11-12T09:29:01,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:01,264 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-12T09:29:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-12T09:29:01,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/121c01cac3884fb7bafca1a28bc15718 is 49, key is activesession:/session:c/1699781340612/Put/seqid=0 2023-11-12T09:29:01,689 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:01,698 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/121c01cac3884fb7bafca1a28bc15718 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:01,707 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, entries=1, sequenceid=5, filesize=4.9 K 2023-11-12T09:29:01,710 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 446ms, sequenceid=5, compaction requested=false 2023-11-12T09:29:01,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:29:01,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2520): Flush status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:01,711 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. for snapshot_backup_system completed. 2023-11-12T09:29:01,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:01,719 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:01,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:29:01,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718] hfiles 2023-11-12T09:29:01,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 for snapshot=snapshot_backup_system 2023-11-12T09:29:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-12T09:29:02,143 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:02,147 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=38 2023-11-12T09:29:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=38 2023-11-12T09:29:02,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:02,152 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:02,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-11-12T09:29:02,155 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:02,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 in 1.0590 sec 2023-11-12T09:29:02,157 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:02,160 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:02,160 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:02,163 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:02,184 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:02,197 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:02,197 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:02,199 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:02,199 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-11-12T09:29:02,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5490 sec 2023-11-12T09:29:02,765 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-12T09:29:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-12T09:29:02,768 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 37 completed 2023-11-12T09:29:02,769 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(115): Backup backup_1699781336188 started at 1699781342768. 2023-11-12T09:29:02,791 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(119): Backup session backup_1699781336188 has been started. 2023-11-12T09:29:02,794 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:29:02,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:02,809 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:02,809 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:02,810 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:02,810 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:02,811 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,811 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:02,812 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:29:02,812 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,812 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:02,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:02,813 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,813 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:02,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:02,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:02,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:02,814 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:02,816 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:02,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:02,816 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:02,816 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:02,817 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:02,817 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:02,817 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:02,820 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,820 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:02,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:02,820 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,820 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:02,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:02,822 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:02,822 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5d533241[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:02,822 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:02,823 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,826 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,826 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,826 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:02,826 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,826 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:02,827 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:02,827 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,828 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781318968 highest: 1699781321500 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,828 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0:(num 1699781318968) roll requested 2023-11-12T09:29:02,839 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828, exclude list is [], retry=0 2023-11-12T09:29:02,843 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:02,846 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781318968 with entries=18, filesize=3.96 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:02,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:02,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781318968 is not closed yet, will try archiving it next time 2023-11-12T09:29:02,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1:(num 1699781321500) roll requested 2023-11-12T09:29:02,860 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848, exclude list is [], retry=0 2023-11-12T09:29:02,865 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:02,867 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781321500 with entries=11, filesize=2.04 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:02,867 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:02,867 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781321500 is not closed yet, will try archiving it next time 2023-11-12T09:29:02,867 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta:.meta(num 1699781319976) roll requested 2023-11-12T09:29:02,879 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta, exclude list is [], retry=0 2023-11-12T09:29:02,883 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:02,885 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta with entries=40, filesize=12.94 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta 2023-11-12T09:29:02,885 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:02,885 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:02,889 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-11-12T09:29:02,889 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781342828 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,889 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:02,891 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:50602, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:29:02,909 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:02,909 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37291,1699781316827' in zk 2023-11-12T09:29:02,912 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,912 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:02,912 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:02,912 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:02,912 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:02,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:02,914 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:02,914 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:02,914 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:02,915 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,915 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:02,915 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:02,916 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,916 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37291,1699781316827': 2023-11-12T09:29:02,916 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:02,917 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:02,917 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:02,917 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:02,917 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:02,917 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:02,919 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,919 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,919 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:29:02,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,919 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:02,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:02,920 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:02,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:02,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:02,920 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:02,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:02,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:02,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,922 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:02,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:02,923 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,931 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,931 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:02,931 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:02,932 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:02,932 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:29:02,932 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,932 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:02,932 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:02,932 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37291-0x1004cf66f250001, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:29:02,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:02,932 DEBUG [Listener at localhost/33293-EventThread {}] zookeeper.ZKWatcher(604): master:43745-0x1004cf66f250000, quorum=127.0.0.1:57937, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:02,932 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:02,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:02,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:02,933 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:02,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:02,935 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:29:02,935 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:29:03,035 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:29:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:03,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } 2023-11-12T09:29:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781343050 (current time:1699781343050). 2023-11-12T09:29:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781343048_default_test-1699781326335 VERSION not specified, setting to 2 2023-11-12T09:29:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x574faef5 to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:29:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e591e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:03,061 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x574faef5 to 127.0.0.1:57937 2023-11-12T09:29:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } 2023-11-12T09:29:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-11-12T09:29:03,066 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:29:03,067 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:03,071 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_1699781343048_default_test-1699781326335 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:03,072 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:29:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:29:03,487 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:03,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 355fc0ccb227f73de53d754779d91940}] 2023-11-12T09:29:03,489 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 355fc0ccb227f73de53d754779d91940 2023-11-12T09:29:03,640 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:03,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2023-11-12T09:29:03,641 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:29:03,643 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2819): Flushing 355fc0ccb227f73de53d754779d91940 2/2 column families, dataSize=324 B heapSize=1.25 KB 2023-11-12T09:29:03,658 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f3/056fb51661cb42c99d46388c5abd0df9 is 58, key is row-test-1699781326335-3-0/f3:q1/1699781336177/Put/seqid=0 2023-11-12T09:29:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:29:04,068 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f3/056fb51661cb42c99d46388c5abd0df9 2023-11-12T09:29:04,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f3/056fb51661cb42c99d46388c5abd0df9 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f3/056fb51661cb42c99d46388c5abd0df9 2023-11-12T09:29:04,085 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f3/056fb51661cb42c99d46388c5abd0df9, entries=6, sequenceid=115, filesize=5.2 K 2023-11-12T09:29:04,087 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~1008 B/1008, currentSize=0 B/0 for 355fc0ccb227f73de53d754779d91940 in 444ms, sequenceid=115, compaction requested=false 2023-11-12T09:29:04,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2520): Flush status journal for 355fc0ccb227f73de53d754779d91940: 2023-11-12T09:29:04,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. for snapshot_1699781343048_default_test-1699781326335 completed. 2023-11-12T09:29:04,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(240): Storing 'test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940.' region-info for snapshot=snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:04,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:04,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250] hfiles 2023-11-12T09:29:04,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250 for snapshot=snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:04,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f3/056fb51661cb42c99d46388c5abd0df9] hfiles 2023-11-12T09:29:04,088 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f3/056fb51661cb42c99d46388c5abd0df9 for snapshot=snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:29:04,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:29:04,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=40 2023-11-12T09:29:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=40 2023-11-12T09:29:04,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781343048_default_test-1699781326335 on region 355fc0ccb227f73de53d754779d91940 2023-11-12T09:29:04,500 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 355fc0ccb227f73de53d754779d91940 2023-11-12T09:29:04,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-12T09:29:04,503 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:04,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; SnapshotRegionProcedure 355fc0ccb227f73de53d754779d91940 in 1.0140 sec 2023-11-12T09:29:04,504 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:04,505 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:04,505 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:04,506 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:04,929 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:04,937 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:04,938 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_1699781343048_default_test-1699781326335 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:04,939 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:04,940 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-11-12T09:29:04,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1699781343048_default_test-1699781326335 table=test-1699781326335 type=FLUSH ttl=0 } in 1.8770 sec 2023-11-12T09:29:05,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-12T09:29:05,174 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699781326335, procId: 39 completed 2023-11-12T09:29:05,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } 2023-11-12T09:29:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781345175 (current time:1699781345175). 2023-11-12T09:29:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781345174_ns2_test-16997813263351 VERSION not specified, setting to 2 2023-11-12T09:29:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x69f8ba55 to 127.0.0.1:57937 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:29:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436a8999, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:05,184 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:29:05,184 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:29:05,185 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:29:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:05,185 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-12T09:29:05,188 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x69f8ba55 to 127.0.0.1:57937 2023-11-12T09:29:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } 2023-11-12T09:29:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-11-12T09:29:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:29:05,193 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:05,194 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:05,197 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_1699781345174_ns2_test-16997813263351 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:05,199 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:05,209 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:05,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 7a46314c3b11bd429216bed3cf7244cb}] 2023-11-12T09:29:05,211 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:29:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:29:05,362 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:05,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=42 2023-11-12T09:29:05,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:29:05,364 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2819): Flushing 7a46314c3b11bd429216bed3cf7244cb 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:29:05,381 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/.tmp/f/827070b5b9a44bdca95f468a70b8e7aa is 37, key is row10/f:q1/1699781330183/Put/seqid=0 2023-11-12T09:29:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:29:05,789 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/.tmp/f/827070b5b9a44bdca95f468a70b8e7aa 2023-11-12T09:29:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:29:05,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/.tmp/f/827070b5b9a44bdca95f468a70b8e7aa as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/827070b5b9a44bdca95f468a70b8e7aa 2023-11-12T09:29:05,805 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/827070b5b9a44bdca95f468a70b8e7aa, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:29:05,806 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 7a46314c3b11bd429216bed3cf7244cb in 442ms, sequenceid=103, compaction requested=false 2023-11-12T09:29:05,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2520): Flush status journal for 7a46314c3b11bd429216bed3cf7244cb: 2023-11-12T09:29:05,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. for snapshot_1699781345174_ns2_test-16997813263351 completed. 2023-11-12T09:29:05,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb.' region-info for snapshot=snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:05,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:05,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/827070b5b9a44bdca95f468a70b8e7aa] hfiles 2023-11-12T09:29:05,808 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/827070b5b9a44bdca95f468a70b8e7aa for snapshot=snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:06,218 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:29:06,218 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=42 2023-11-12T09:29:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=42 2023-11-12T09:29:06,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781345174_ns2_test-16997813263351 on region 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:29:06,220 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:29:06,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-11-12T09:29:06,223 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:06,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; SnapshotRegionProcedure 7a46314c3b11bd429216bed3cf7244cb in 1.0120 sec 2023-11-12T09:29:06,223 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:06,224 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:06,224 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:06,225 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:29:06,644 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:06,651 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:06,652 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_1699781345174_ns2_test-16997813263351 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:06,653 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:06,653 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-11-12T09:29:06,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1699781345174_ns2_test-16997813263351 table=ns2:test-16997813263351 type=FLUSH ttl=0 } in 1.4630 sec 2023-11-12T09:29:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:29:07,301 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns2:test-16997813263351, procId: 41 completed 2023-11-12T09:29:07,301 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781336188 2023-11-12T09:29:07,301 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:29:07,303 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:29:07,305 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781336188_ns2:test-16997813263351 2023-11-12T09:29:07,305 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781345174_ns2_test-16997813263351 to hdfs://localhost:36805/backupUT/backup_1699781336188/ns2/test-16997813263351/ 2023-11-12T09:29:07,315 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:29:07,345 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36805, inputRoot=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:29:07,345 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43745,1699781313745_-460890622_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36805/backupUT/backup_1699781336188/ns2/test-16997813263351, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36805/backupUT/backup_1699781336188/ns2/test-16997813263351/.hbase-snapshot/.tmp/snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:07,345 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36805, inputRoot=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c, snapshotDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_1699781345174_ns2_test-16997813263351. 2023-11-12T09:29:07,357 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_1699781345174_ns2_test-16997813263351 to hdfs://localhost:36805/backupUT/backup_1699781336188/ns2/test-16997813263351/.hbase-snapshot/.tmp/snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:07,792 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:07,793 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:07,794 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:07,794 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,453 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:29:08,457 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57064, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:29:08,800 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-736760488515877976.jar 2023-11-12T09:29:08,801 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,801 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,802 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,802 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,803 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,803 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,804 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,805 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,805 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:08,806 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:08,807 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:08,807 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:08,808 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:08,808 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:08,809 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:08,809 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:08,810 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:08,810 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:08,811 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:08,811 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:08,812 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:08,816 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:08,817 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:08,817 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:08,818 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:08,818 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:08,819 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:08,819 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:08,903 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:29:08,925 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781345174_ns2_test-16997813263351' hfile list 2023-11-12T09:29:08,935 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:29:09,145 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781349098/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:09,145 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781349098/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:09,303 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:29:09,334 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/827070b5b9a44bdca95f468a70b8e7aa output=hdfs://localhost:36805/backupUT/backup_1699781336188/ns2/test-16997813263351/archive/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/827070b5b9a44bdca95f468a70b8e7aa 2023-11-12T09:29:09,334 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-12T09:29:10,207 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:10,208 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:10,216 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:10,216 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781345174_ns2_test-16997813263351 finished. 2023-11-12T09:29:10,216 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781336188_test-1699781326335 2023-11-12T09:29:10,216 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781343048_default_test-1699781326335 to hdfs://localhost:36805/backupUT/backup_1699781336188/default/test-1699781326335/ 2023-11-12T09:29:10,216 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:29:10,241 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36805, inputRoot=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c 2023-11-12T09:29:10,241 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43745,1699781313745_-109025432_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36805/backupUT/backup_1699781336188/default/test-1699781326335, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36805/backupUT/backup_1699781336188/default/test-1699781326335/.hbase-snapshot/.tmp/snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:10,241 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36805, inputRoot=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c, snapshotDir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_1699781343048_default_test-1699781326335. 2023-11-12T09:29:10,252 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_1699781343048_default_test-1699781326335 to hdfs://localhost:36805/backupUT/backup_1699781336188/default/test-1699781326335/.hbase-snapshot/.tmp/snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:10,670 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:10,671 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:10,671 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:10,672 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,635 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-7089734908991443238.jar 2023-11-12T09:29:11,636 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,636 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,637 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,637 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,638 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,638 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,638 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,639 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,639 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:11,640 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:11,641 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:11,641 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:11,641 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:11,642 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:11,643 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:11,643 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:11,643 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:11,644 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:11,644 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:11,645 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:11,645 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:11,646 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:11,646 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:11,647 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:11,647 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:11,648 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:11,648 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:11,649 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:11,676 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:29:11,693 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781343048_default_test-1699781326335' hfile list 2023-11-12T09:29:11,696 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(674): export split=0 size=13.4 K 2023-11-12T09:29:11,773 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781351738/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:11,773 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781351738/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:11,827 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:29:11,853 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250 output=hdfs://localhost:36805/backupUT/backup_1699781336188/default/test-1699781326335/archive/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/1f211f4489a24367a0d18ad65d845250 2023-11-12T09:29:11,853 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-12T09:29:12,287 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f3/056fb51661cb42c99d46388c5abd0df9 output=hdfs://localhost:36805/backupUT/backup_1699781336188/default/test-1699781326335/archive/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f3/056fb51661cb42c99d46388c5abd0df9 2023-11-12T09:29:12,287 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=5370 (5.2 K) time=0sec 5.121M/sec 2023-11-12T09:29:12,812 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:12,813 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:12,822 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:12,822 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781343048_default_test-1699781326335 finished. 2023-11-12T09:29:12,822 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1034): ns2:test-16997813263351 2023-11-12T09:29:12,823 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1034): test-1699781326335 2023-11-12T09:29:12,905 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781336188 2023-11-12T09:29:12,905 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:13,318 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36805/backupUT/backup_1699781336188/.backup.manifest 2023-11-12T09:29:13,318 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781336188 2023-11-12T09:29:13,318 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:13,724 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36805/backupUT/backup_1699781336188/.backup.manifest 2023-11-12T09:29:13,724 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(358): Backup backup_1699781336188 finished: type=FULL,tablelist=ns2:test-16997813263351;test-1699781326335,targetRootDir=hdfs://localhost:36805/backupUT,startts=1699781342768,completets=1699781352902,bytescopied=0 2023-11-12T09:29:13,724 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:29:13,724 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:13,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781345174_ns2_test-16997813263351" 2023-11-12T09:29:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781345174_ns2_test-16997813263351 2023-11-12T09:29:13,741 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781345174_ns2_test-16997813263351 for backup backup_1699781336188 succeeded. 2023-11-12T09:29:13,741 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:13,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781343048_default_test-1699781326335" 2023-11-12T09:29:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781343048_default_test-1699781326335 2023-11-12T09:29:13,746 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781343048_default_test-1699781326335 for backup backup_1699781336188 succeeded. 2023-11-12T09:29:13,747 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:13,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:13,766 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:13,776 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:13,794 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(376): Backup backup_1699781336188 completed. 2023-11-12T09:29:13,914 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(104): writing 99 rows to test-1699781326335 2023-11-12T09:29:13,930 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(108): written 99 rows to test-1699781326335 2023-11-12T09:29:13,946 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(120): written 5 rows to ns2:test-16997813263351 2023-11-12T09:29:13,947 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 0 2023-11-12T09:29:13,989 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c1d23a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:13,990 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:13,991 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57066, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:13,995 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:13,997 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:14,054 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:14,057 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:29:14,058 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(240): BackupId=backup_1699781354023,startts=0,failedts=1699781354058,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-11-12T09:29:14,058 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:29:14,060 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1550): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-11-12T09:29:14,060 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:14,062 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-11-12T09:29:14,062 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-12T09:29:14,064 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:14,065 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(246): Backup backup_1699781354023 failed. Backup session finished. Status: FAILURE 2023-11-12T09:29:14,066 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:14,066 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] ... 41 more 2023-11-12T09:29:14,072 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 1 2023-11-12T09:29:14,112 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdc02b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:14,113 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:14,115 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:14,119 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:14,120 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38948, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:14,172 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:14,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781354175 (current time:1699781354175). 2023-11-12T09:29:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d047bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:14,187 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38962, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-12T09:29:14,192 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:14,193 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:14,195 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:14,198 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:14,605 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:14,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18}] 2023-11-12T09:29:14,607 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:14,758 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:14,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-11-12T09:29:14,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:14,759 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=1.83 KB heapSize=3.35 KB 2023-11-12T09:29:14,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/d4112b1d553941878d79afd85b34fccb is 171, key is trslm:hdfs://localhost:36805/backupUT\x00ns2:test-16997813263351/meta:log-roll-map/1699781352892/Put/seqid=0 2023-11-12T09:29:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:15,179 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:15,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/2b2d2630865e4cd189ee99eefe559197 is 459, key is session:backup_1699781336188/session:context/1699781353775/Put/seqid=0 2023-11-12T09:29:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:15,596 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.07 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:15,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/d4112b1d553941878d79afd85b34fccb as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:15,611 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb, entries=6, sequenceid=20, filesize=5.8 K 2023-11-12T09:29:15,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/2b2d2630865e4cd189ee99eefe559197 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:15,617 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197, entries=3, sequenceid=20, filesize=5.7 K 2023-11-12T09:29:15,618 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.83 KB/1879, heapSize ~3.32 KB/3400, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 859ms, sequenceid=20, compaction requested=false 2023-11-12T09:29:15,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:15,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. for snapshot_backup_system completed. 2023-11-12T09:29:15,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:15,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:15,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:15,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb for snapshot=snapshot_backup_system 2023-11-12T09:29:15,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197] hfiles 2023-11-12T09:29:15,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 for snapshot=snapshot_backup_system 2023-11-12T09:29:15,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 for snapshot=snapshot_backup_system 2023-11-12T09:29:16,024 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:16,025 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-11-12T09:29:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=44 2023-11-12T09:29:16,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:16,026 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:16,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-11-12T09:29:16,029 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:16,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 in 1.4220 sec 2023-11-12T09:29:16,029 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:16,030 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:16,030 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:16,031 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:16,442 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:16,451 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:16,452 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:16,457 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:16,457 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-12T09:29:16,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2680 sec 2023-11-12T09:29:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:29:18,299 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(115): Backup backup_1699781354147 started at 1699781358299. 2023-11-12T09:29:18,301 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(119): Backup session backup_1699781354147 has been started. 2023-11-12T09:29:18,301 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:29:18,302 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(240): BackupId=backup_1699781354147,startts=1699781358299,failedts=1699781358302,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-11-12T09:29:18,302 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:29:18,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable backup:system 2023-11-12T09:29:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-12T09:29:18,325 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781358325"}]},"ts":"1699781358325"} 2023-11-12T09:29:18,327 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-12T09:29:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:29:18,330 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-12T09:29:18,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN}] 2023-11-12T09:29:18,333 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN 2023-11-12T09:29:18,333 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:18,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:29:18,486 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:18,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(119): Close 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:18,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1668): Closing 2bd6ff45da8970c8169b00517ffa7e18, disabling compactions & flushes 2023-11-12T09:29:18,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1690): Closing region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:18,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:18,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. after waiting 0 ms 2023-11-12T09:29:18,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:18,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=214 B heapSize=800 B 2023-11-12T09:29:18,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/69994641b8234d678a7732bb33d85662 is 218, key is session:backup_1699781354147/session:context/1699781358300/Put/seqid=0 2023-11-12T09:29:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:29:18,897 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/69994641b8234d678a7732bb33d85662 2023-11-12T09:29:18,904 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/69994641b8234d678a7732bb33d85662 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/69994641b8234d678a7732bb33d85662 2023-11-12T09:29:18,910 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/69994641b8234d678a7732bb33d85662, entries=1, sequenceid=24, filesize=5.2 K 2023-11-12T09:29:18,911 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 424ms, sequenceid=24, compaction requested=true 2023-11-12T09:29:18,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-11-12T09:29:18,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:18,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1909): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:18,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1622): Region close journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:18,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(163): Closed 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:18,925 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSED 2023-11-12T09:29:18,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-11-12T09:29:18,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 591 msec 2023-11-12T09:29:18,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-12T09:29:18,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN in 597 msec 2023-11-12T09:29:18,931 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781358930"}]},"ts":"1699781358930"} 2023-11-12T09:29:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:29:18,932 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-12T09:29:18,934 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-12T09:29:18,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; DisableTableProcedure table=backup:system in 620 msec 2023-11-12T09:29:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-12T09:29:19,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781359442 (current time:1699781359442). 2023-11-12T09:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1699781359441 VERSION not specified, setting to 2 2023-11-12T09:29:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156d3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:19,451 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49876, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-11-12T09:29:19,455 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:29:19,456 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:19,459 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781359441 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:19,462 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:29:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:29:19,868 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-12T09:29:19,870 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-12T09:29:19,870 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-12T09:29:19,870 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:19,871 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:19,871 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:19,871 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:19,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/69994641b8234d678a7732bb33d85662] hfiles 2023-11-12T09:29:19,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:19,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:19,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/69994641b8234d678a7732bb33d85662 2023-11-12T09:29:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:29:20,280 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-12T09:29:20,281 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:20,281 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:20,281 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1699781359441 2023-11-12T09:29:20,282 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781359441 2023-11-12T09:29:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:29:20,694 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:20,702 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:20,703 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781359441 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1699781359441 2023-11-12T09:29:20,704 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:20,705 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-11-12T09:29:20,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781359441 table=backup:system type=FLUSH ttl=0 } in 1.2530 sec 2023-11-12T09:29:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:29:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781354175 type: FLUSH version: 2 ttl: 0 ) 2023-11-12T09:29:21,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-12T09:29:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:29:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:29:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:29:22,005 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-11-12T09:29:22,006 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-12T09:29:22,007 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:22,013 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781354175 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:22,013 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system 2023-11-12T09:29:22,016 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-12T09:29:22,017 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,022 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:22,031 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/69994641b8234d678a7732bb33d85662 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/69994641b8234d678a7732bb33d85662 2023-11-12T09:29:22,032 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:22,032 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781354175 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:22,033 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781362033"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:22,036 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:22,036 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:22,036 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781362034"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781362034"}]},"ts":"1699781362034"} 2023-11-12T09:29:22,037 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:22,037 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-12T09:29:22,037 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:22,038 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-12T09:29:22,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781354175 type: FLUSH version: 2 ttl: 0 ) in 460 msec 2023-11-12T09:29:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:29:22,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699781359441" 2023-11-12T09:29:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1699781359441 2023-11-12T09:29:22,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$12(2662): Client=jenkins//172.31.12.81 enable backup:system 2023-11-12T09:29:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-12T09:29:22,221 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-12T09:29:22,221 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781362221"}]},"ts":"1699781362221"} 2023-11-12T09:29:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:29:22,222 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:29:22,229 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-12T09:29:22,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN}] 2023-11-12T09:29:22,230 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN 2023-11-12T09:29:22,231 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-12T09:29:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:29:22,383 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:22,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:22,484 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE; InitializeStoreFileTrackerProcedure table=test-1699781326335 2023-11-12T09:29:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:29:22,536 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:22,540 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(130): Open backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:22,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7254): Opening region: {ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:22,540 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:22,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(888): Instantiated backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:22,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7296): checking encryption for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7299): checking classloading for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,542 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,543 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName meta 2023-11-12T09:29:22,543 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:22,549 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:22,549 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:22,549 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,550 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName session 2023-11-12T09:29:22,550 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:22,556 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:22,561 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:22,561 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:22,562 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,564 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:29:22,565 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1076): writing seq id for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:22,566 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): Opened 2bd6ff45da8970c8169b00517ffa7e18; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10570648160, jitterRate=-0.015531674027442932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:29:22,567 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(992): Region open journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:22,567 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., pid=52, masterSystemTime=1699781362536 2023-11-12T09:29:22,569 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:22,569 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(158): Opened backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:22,569 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:22,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-11-12T09:29:22,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 187 msec 2023-11-12T09:29:22,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-12T09:29:22,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN in 343 msec 2023-11-12T09:29:22,574 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781362574"}]},"ts":"1699781362574"} 2023-11-12T09:29:22,575 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:29:22,577 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-12T09:29:22,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; EnableTableProcedure table=backup:system in 366 msec 2023-11-12T09:29:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:29:22,825 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-12T09:29:22,825 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:22,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:22,834 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:22,834 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-12T09:29:22,836 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:22,842 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(246): Backup backup_1699781354147 failed. Backup session finished. Status: FAILURE 2023-11-12T09:29:22,842 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:22,842 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] ... 41 more 2023-11-12T09:29:22,848 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 2 2023-11-12T09:29:22,866 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4e124e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:22,867 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:22,869 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:22,875 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:22,876 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:22,900 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/.tabledesc/.tableinfo.0000000002.598 2023-11-12T09:29:22,900 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/.tabledesc/.tableinfo.0000000003.640 2023-11-12T09:29:22,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; InitializeStoreFileTrackerProcedure table=test-1699781326335 in 423 msec 2023-11-12T09:29:22,947 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:22,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781362952 (current time:1699781362952). 2023-11-12T09:29:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ed2ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:22,964 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49884, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-11-12T09:29:22,970 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:29:22,971 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:22,974 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:22,976 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:22,988 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:22,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18}] 2023-11-12T09:29:22,990 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:29:23,140 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:23,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-11-12T09:29:23,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:23,143 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-11-12T09:29:23,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/47fb20b82eee4f119ab997125e5b50b8 is 256, key is session:backup_1699781354147/session:context/1699781362834/Put/seqid=0 2023-11-12T09:29:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:29:23,562 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/47fb20b82eee4f119ab997125e5b50b8 2023-11-12T09:29:23,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/47fb20b82eee4f119ab997125e5b50b8 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8 2023-11-12T09:29:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:29:23,575 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8, entries=2, sequenceid=33, filesize=5.2 K 2023-11-12T09:29:23,576 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 434ms, sequenceid=33, compaction requested=true 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. for snapshot_backup_system completed. 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:23,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb for snapshot=snapshot_backup_system 2023-11-12T09:29:23,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8] hfiles 2023-11-12T09:29:23,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 for snapshot=snapshot_backup_system 2023-11-12T09:29:23,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 for snapshot=snapshot_backup_system 2023-11-12T09:29:23,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8 for snapshot=snapshot_backup_system 2023-11-12T09:29:23,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:23,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-11-12T09:29:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=55 2023-11-12T09:29:23,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:23,984 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:23,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-11-12T09:29:23,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 in 997 msec 2023-11-12T09:29:23,987 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:23,987 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:23,988 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:23,988 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:23,989 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:29:24,402 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:24,410 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:24,410 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:24,411 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:24,412 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-11-12T09:29:24,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4450 sec 2023-11-12T09:29:24,480 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:29:25,077 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(115): Backup backup_1699781362909 started at 1699781365077. 2023-11-12T09:29:25,079 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(119): Backup session backup_1699781362909 has been started. 2023-11-12T09:29:25,080 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [test-1699781326335, ns2:test-16997813263351] 2023-11-12T09:29:25,083 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(76): StartCode 1699781321500for backupID backup_1699781362909 2023-11-12T09:29:25,083 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:29:25,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:25,084 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:25,084 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:25,086 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:25,086 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:25,086 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:25,088 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,088 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:25,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:25,088 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:25,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:25,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:25,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:25,089 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:25,089 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:25,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:25,089 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:25,089 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:25,090 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:25,090 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:25,090 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:25,093 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:25,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:25,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:25,093 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:25,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:25,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:25,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:25,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:25,095 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:25,095 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:25,095 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@45634d98[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:25,095 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:25,098 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:25,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:25,098 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:25,098 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:25,098 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:25,098 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,098 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781342828 highest: 1699781342848 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,098 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0:(num 1699781342828) roll requested 2023-11-12T09:29:25,110 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098, exclude list is [], retry=0 2023-11-12T09:29:25,114 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:25,116 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 with entries=124, filesize=18.41 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 2023-11-12T09:29:25,116 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:25,116 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 is not closed yet, will try archiving it next time 2023-11-12T09:29:25,117 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1:(num 1699781342848) roll requested 2023-11-12T09:29:25,117 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781318968 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781318968 2023-11-12T09:29:25,128 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117, exclude list is [], retry=0 2023-11-12T09:29:25,132 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:25,134 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 2023-11-12T09:29:25,134 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:25,134 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 is not closed yet, will try archiving it next time 2023-11-12T09:29:25,134 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta:.meta(num 1699781342867) roll requested 2023-11-12T09:29:25,146 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta, exclude list is [], retry=0 2023-11-12T09:29:25,150 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:25,152 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta 2023-11-12T09:29:25,152 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:25,152 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:25,155 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta not finished, retry = 0 2023-11-12T09:29:25,159 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-11-12T09:29:25,159 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781365098 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,168 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:25,168 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37291,1699781316827' in zk 2023-11-12T09:29:25,172 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:25,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,172 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:25,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:25,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:25,172 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:25,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:25,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:25,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:25,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:25,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:25,176 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37291,1699781316827': 2023-11-12T09:29:25,177 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:25,177 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:25,177 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:25,177 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:25,177 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:25,177 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:25,179 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:25,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:25,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:25,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:25,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:25,180 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:25,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:25,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:25,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:25,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:25,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:25,181 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:25,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:25,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:25,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:25,187 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:25,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:25,188 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:25,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:25,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:25,188 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:25,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:25,292 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37291=1699781321500} newestTimestamps: {jenkins-hbase3.apache.org:37291=1699781342848} 2023-11-12T09:29:25,294 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta 2023-11-12T09:29:25,295 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta 2023-11-12T09:29:25,295 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta 2023-11-12T09:29:25,295 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:25,295 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 2023-11-12T09:29:25,295 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:25,295 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 2023-11-12T09:29:25,707 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997813263351 target: hdfs://localhost:36805/backupUT/backup_1699781362909/ns2/test-16997813263351 descriptor: 'ns2:test-16997813263351', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:25,707 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:29:25,709 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997813263351 2023-11-12T09:29:26,115 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997813263351 2023-11-12T09:29:26,528 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781326335 target: hdfs://localhost:36805/backupUT/backup_1699781362909/default/test-1699781326335 descriptor: 'test-1699781326335', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:26,528 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:29:26,531 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(152): Starting to write region info for table test-1699781326335 2023-11-12T09:29:26,936 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(159): Finished writing region info for table test-1699781326335 2023-11-12T09:29:26,946 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:36805/backupUT/.tmp/backup_1699781362909 from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:26,956 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301629c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:26,959 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:26,960 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48496, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:26,976 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:29:26,976 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781326335 2023-11-12T09:29:26,981 DEBUG [hconnection-0x2fae0ef1-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:26,983 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59436, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:26,986 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781326335: test-1699781326335; 2023-11-12T09:29:26,987 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997813263351 2023-11-12T09:29:26,990 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997813263351: ns2:test-16997813263351; 2023-11-12T09:29:26,990 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:29:26,990 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/hbase-staging/partitions_6d24aa31-f01a-42f6-89bf-f8715ffcd7b6 2023-11-12T09:29:27,433 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:27,433 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:27,434 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:27,434 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,397 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-5827546522130128108.jar 2023-11-12T09:29:28,397 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,398 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,398 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,399 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,399 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,399 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,400 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,400 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,401 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,401 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:28,402 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:28,402 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:28,403 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:28,403 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:28,404 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:28,404 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:28,405 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:28,405 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:28,406 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:28,406 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:28,407 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:28,409 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,409 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,410 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,410 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,410 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,411 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:28,411 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:28,412 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781326335,ns2:test-16997813263351 2023-11-12T09:29:28,412 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:28,413 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:29,363 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-1042910360380099688.jar 2023-11-12T09:29:29,384 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 for WAL files 2023-11-12T09:29:29,386 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:29,386 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 for WAL files 2023-11-12T09:29:29,387 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:29,473 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781369437/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:29,473 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781369437/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:29,557 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 (-9223372036854775808:9223372036854775807) length:18860 2023-11-12T09:29:29,574 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:29,574 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:29,617 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 (-9223372036854775808:9223372036854775807) length:1198 2023-11-12T09:29:29,621 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:29,621 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:29,750 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5019b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:29,752 DEBUG [hconnection-0x1375c8b5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:29,755 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59440, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:29,758 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:29,760 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-12T09:29:29,760 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:29,768 DEBUG [pool-226-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/backupUT/.tmp/backup_1699781362909/_temporary/0/_temporary/attempt_local765502292_0003_r_000000_0/ns2/test-16997813263351/f/1ae1b2f399d04815bf78cf8cbe463e84 is 38, key is row-t20/f:q1/1699781353933/Put/seqid=0 2023-11-12T09:29:29,771 WARN [Thread-914 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37291] Specified favored nodes: [jenkins-hbase3.apache.org:37291] 2023-11-12T09:29:30,208 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12830690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:30,210 DEBUG [hconnection-0x34640e82-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:30,211 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:30,213 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:30,214 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1699781326335-1-0] 2023-11-12T09:29:30,214 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:30,231 DEBUG [pool-226-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/backupUT/.tmp/backup_1699781362909/_temporary/0/_temporary/attempt_local765502292_0003_r_000001_0/default/test-1699781326335/f/5b17f281e046436d8b2e8fde0e0ba01b is 59, key is row-test-1699781326335-1-10/f:q1/1699781353824/Put/seqid=0 2023-11-12T09:29:30,233 WARN [Thread-924 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37291] Specified favored nodes: [jenkins-hbase3.apache.org:37291] 2023-11-12T09:29:30,512 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:36805/backupUT 2023-11-12T09:29:30,512 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781362909 2023-11-12T09:29:30,512 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:29:30,527 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:36805/backupUT/.tmp/backup_1699781362909, hdfs://localhost:36805/backupUT] 2023-11-12T09:29:31,667 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:29:31,669 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781362909 2023-11-12T09:29:31,669 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-11-12T09:29:31,670 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local2036829249_0004 completed: true true 2023-11-12T09:29:31,675 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3792131 FILE: Number of bytes written=5082281 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=189984 HDFS: Number of bytes written=2364105 HDFS: Number of read operations=690 HDFS: Number of large read operations=0 HDFS: Number of write operations=247 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1665138688 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-11-12T09:29:31,675 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:36805/backupUT/.tmp/backup_1699781362909 to hdfs://localhost:36805/backupUT finished. 2023-11-12T09:29:31,677 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:29:31,677 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(240): BackupId=backup_1699781362909,startts=1699781365077,failedts=1699781371677,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_2 in testing 2023-11-12T09:29:31,677 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:29:31,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable backup:system 2023-11-12T09:29:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-12T09:29:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:29:31,687 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781371687"}]},"ts":"1699781371687"} 2023-11-12T09:29:31,689 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-12T09:29:31,691 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-12T09:29:31,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN}] 2023-11-12T09:29:31,692 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN 2023-11-12T09:29:31,693 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:31,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:31,728 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-12T09:29:31,728 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-12T09:29:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:29:31,845 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:31,846 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(119): Close 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:31,846 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1668): Closing 2bd6ff45da8970c8169b00517ffa7e18, disabling compactions & flushes 2023-11-12T09:29:31,846 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1690): Closing region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:31,846 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:31,846 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. after waiting 0 ms 2023-11-12T09:29:31,846 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:31,846 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=540 B heapSize=1.24 KB 2023-11-12T09:29:31,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/6f2db6b4bd5645f397177d137d0d3463 is 116, key is rslogts:hdfs://localhost:36805/backupUT\x00jenkins-hbase3.apache.org:37291/meta:rs-log-ts/1699781365167/Put/seqid=0 2023-11-12T09:29:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:29:32,264 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=112 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/6f2db6b4bd5645f397177d137d0d3463 2023-11-12T09:29:32,273 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/efe0d88673f542b9bb79971c221835e3 is 218, key is session:backup_1699781362909/session:context/1699781371668/Put/seqid=0 2023-11-12T09:29:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:29:32,679 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/efe0d88673f542b9bb79971c221835e3 2023-11-12T09:29:32,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/6f2db6b4bd5645f397177d137d0d3463 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/6f2db6b4bd5645f397177d137d0d3463 2023-11-12T09:29:32,693 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/6f2db6b4bd5645f397177d137d0d3463, entries=1, sequenceid=39, filesize=5.3 K 2023-11-12T09:29:32,694 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/efe0d88673f542b9bb79971c221835e3 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/efe0d88673f542b9bb79971c221835e3 2023-11-12T09:29:32,701 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/efe0d88673f542b9bb79971c221835e3, entries=1, sequenceid=39, filesize=5.2 K 2023-11-12T09:29:32,702 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(3022): Finished flush of dataSize ~540 B/540, heapSize ~1.21 KB/1240, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 856ms, sequenceid=39, compaction requested=true 2023-11-12T09:29:32,715 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=27 2023-11-12T09:29:32,716 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:32,717 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1909): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:32,717 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1622): Region close journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:32,719 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(163): Closed 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:32,720 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSED 2023-11-12T09:29:32,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-11-12T09:29:32,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 1.0270 sec 2023-11-12T09:29:32,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-11-12T09:29:32,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN in 1.0320 sec 2023-11-12T09:29:32,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781372725"}]},"ts":"1699781372725"} 2023-11-12T09:29:32,727 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-12T09:29:32,729 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-12T09:29:32,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0470 sec 2023-11-12T09:29:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:29:32,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781372801 (current time:1699781372801). 2023-11-12T09:29:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1699781372801 VERSION not specified, setting to 2 2023-11-12T09:29:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d110d05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:32,813 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59456, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-11-12T09:29:32,817 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-12T09:29:32,818 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:32,821 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781372801 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:32,823 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-12T09:29:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-12T09:29:33,229 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-12T09:29:33,232 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-12T09:29:33,232 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-12T09:29:33,232 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:33,234 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/6f2db6b4bd5645f397177d137d0d3463, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:33,234 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/6f2db6b4bd5645f397177d137d0d3463 2023-11-12T09:29:33,234 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:33,234 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:33,236 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/efe0d88673f542b9bb79971c221835e3] hfiles 2023-11-12T09:29:33,236 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:33,236 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:33,236 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8 2023-11-12T09:29:33,237 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/efe0d88673f542b9bb79971c221835e3 2023-11-12T09:29:33,243 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-12T09:29:33,244 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:33,244 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:33,244 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1699781372801 2023-11-12T09:29:33,245 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781372801 2023-11-12T09:29:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-12T09:29:33,655 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:33,663 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:33,664 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781372801 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1699781372801 2023-11-12T09:29:33,666 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:33,666 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-11-12T09:29:33,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781372801 table=backup:system type=FLUSH ttl=0 } in 852 msec 2023-11-12T09:29:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-12T09:29:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781362952 type: FLUSH version: 2 ttl: 0 ) 2023-11-12T09:29:33,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-12T09:29:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:29:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:29:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:29:34,349 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-12T09:29:34,349 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-12T09:29:34,350 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:34,353 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781362952 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:34,353 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system 2023-11-12T09:29:34,356 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-12T09:29:34,356 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,363 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/6f2db6b4bd5645f397177d137d0d3463 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/6f2db6b4bd5645f397177d137d0d3463 2023-11-12T09:29:34,363 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:34,367 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/efe0d88673f542b9bb79971c221835e3 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/efe0d88673f542b9bb79971c221835e3 2023-11-12T09:29:34,367 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:34,367 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781362952 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:34,368 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781374368"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:34,370 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:34,370 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:34,371 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781374369"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781374369"}]},"ts":"1699781374369"} 2023-11-12T09:29:34,372 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:34,372 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-12T09:29:34,372 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:34,372 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-12T09:29:34,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781362952 type: FLUSH version: 2 ttl: 0 ) in 439 msec 2023-11-12T09:29:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:29:34,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699781372801" 2023-11-12T09:29:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1699781372801 2023-11-12T09:29:34,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$12(2662): Client=jenkins//172.31.12.81 enable backup:system 2023-11-12T09:29:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=61, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-12T09:29:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-12T09:29:34,550 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-12T09:29:34,550 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781374550"}]},"ts":"1699781374550"} 2023-11-12T09:29:34,551 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:29:34,553 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-12T09:29:34,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN}] 2023-11-12T09:29:34,555 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN 2023-11-12T09:29:34,556 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-12T09:29:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-12T09:29:34,707 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:34,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-12T09:29:34,861 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:34,864 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(130): Open backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:34,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7254): Opening region: {ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:34,864 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:34,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,865 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(888): Instantiated backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:34,865 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7296): checking encryption for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,865 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7299): checking classloading for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,866 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,867 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName meta 2023-11-12T09:29:34,867 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:34,873 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:34,873 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:34,873 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,874 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName session 2023-11-12T09:29:34,874 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:34,880 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:34,882 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:29:34,885 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:34,889 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8 2023-11-12T09:29:34,889 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:34,890 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,891 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,892 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:29:34,894 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1076): writing seq id for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:34,895 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): Opened 2bd6ff45da8970c8169b00517ffa7e18; next sequenceid=43; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11861377600, jitterRate=0.10467687249183655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:29:34,895 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(992): Region open journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:34,896 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., pid=63, masterSystemTime=1699781374860 2023-11-12T09:29:34,898 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(403): Add compact mark for store 2bd6ff45da8970c8169b00517ffa7e18:session, priority=-2147483648, current under compaction store size is 1 2023-11-12T09:29:34,900 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-12T09:29:34,900 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-11-12T09:29:34,901 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:34,901 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(158): Opened backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:34,902 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPEN, openSeqNum=43, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:34,903 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16169 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-11-12T09:29:34,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-11-12T09:29:34,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 194 msec 2023-11-12T09:29:34,905 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HStore(1534): 2bd6ff45da8970c8169b00517ffa7e18/session is initiating minor compaction (all files) 2023-11-12T09:29:34,905 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 2bd6ff45da8970c8169b00517ffa7e18/session in backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:34,905 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8] into tmpdir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp, totalSize=15.8 K 2023-11-12T09:29:34,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=62, resume processing ppid=61 2023-11-12T09:29:34,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, ppid=61, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN in 352 msec 2023-11-12T09:29:34,906 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.Compactor(223): Compacting 121c01cac3884fb7bafca1a28bc15718, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1699781340612 2023-11-12T09:29:34,907 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781374907"}]},"ts":"1699781374907"} 2023-11-12T09:29:34,907 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2b2d2630865e4cd189ee99eefe559197, keycount=3, bloomtype=ROW, size=5.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1699781353775 2023-11-12T09:29:34,907 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.Compactor(223): Compacting 47fb20b82eee4f119ab997125e5b50b8, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1699781362834 2023-11-12T09:29:34,908 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:29:34,910 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-12T09:29:34,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, state=SUCCESS; EnableTableProcedure table=backup:system in 364 msec 2023-11-12T09:29:34,950 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bd6ff45da8970c8169b00517ffa7e18#session#compaction#10 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-11-12T09:29:34,950 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/80629eb81b874242bb02bb184d9fd334 is 459, key is session:backup_1699781336188/session:context/1699781353775/Put/seqid=0 2023-11-12T09:29:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-12T09:29:35,152 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-12T09:29:35,153 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:35,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:35,160 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:35,160 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-11-12T09:29:35,161 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:36805/backupUT/backup_1699781362909/ns2/test-16997813263351 done. 2023-11-12T09:29:35,162 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(208): hdfs://localhost:36805/backupUT/backup_1699781362909/ns2 is empty, remove it. 2023-11-12T09:29:35,163 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:36805/backupUT/backup_1699781362909/default/test-1699781326335 done. 2023-11-12T09:29:35,163 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(208): hdfs://localhost:36805/backupUT/backup_1699781362909/default is empty, remove it. 2023-11-12T09:29:35,165 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:35,171 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(246): Backup backup_1699781362909 failed. Backup session finished. Status: FAILURE 2023-11-12T09:29:35,171 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:35,172 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] ... 41 more 2023-11-12T09:29:35,176 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 3 2023-11-12T09:29:35,197 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265e871e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:35,198 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:35,200 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42734, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:35,203 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:35,204 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35998, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:35,246 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:35,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781375249 (current time:1699781375249). 2023-11-12T09:29:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@353b18bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:35,259 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36014, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-12T09:29:35,265 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:29:35,266 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:35,268 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:35,270 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:35,362 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/80629eb81b874242bb02bb184d9fd334 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 2023-11-12T09:29:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:29:35,378 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:29:35,378 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 2bd6ff45da8970c8169b00517ffa7e18/session of 2bd6ff45da8970c8169b00517ffa7e18 into 80629eb81b874242bb02bb184d9fd334(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-11-12T09:29:35,378 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:35,378 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., storeName=2bd6ff45da8970c8169b00517ffa7e18/session, priority=13, startTime=1699781374898; duration=0sec 2023-11-12T09:29:35,379 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-12T09:29:35,379 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bd6ff45da8970c8169b00517ffa7e18:session 2023-11-12T09:29:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:29:35,677 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:35,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18}] 2023-11-12T09:29:35,679 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:35,830 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:35,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-11-12T09:29:35,831 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:35,831 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-11-12T09:29:35,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/bdb2a855702c40c3814f7cdbdd5eb963 is 256, key is session:backup_1699781362909/session:context/1699781375164/Put/seqid=0 2023-11-12T09:29:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:29:36,240 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:29:36,248 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/bdb2a855702c40c3814f7cdbdd5eb963 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:29:36,253 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963, entries=2, sequenceid=49, filesize=5.2 K 2023-11-12T09:29:36,254 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 423ms, sequenceid=49, compaction requested=false 2023-11-12T09:29:36,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:36,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. for snapshot_backup_system completed. 2023-11-12T09:29:36,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:36,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:36,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:36,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb for snapshot=snapshot_backup_system 2023-11-12T09:29:36,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963] hfiles 2023-11-12T09:29:36,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 for snapshot=snapshot_backup_system 2023-11-12T09:29:36,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 for snapshot=snapshot_backup_system 2023-11-12T09:29:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:29:36,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:36,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-11-12T09:29:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=65 2023-11-12T09:29:36,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:36,666 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:36,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-11-12T09:29:36,668 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:36,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 in 990 msec 2023-11-12T09:29:36,668 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:36,669 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:36,669 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:36,670 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:37,083 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:37,090 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:37,090 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:37,092 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:37,092 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-12T09:29:37,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8320 sec 2023-11-12T09:29:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:29:37,371 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(115): Backup backup_1699781375226 started at 1699781377371. 2023-11-12T09:29:37,373 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(119): Backup session backup_1699781375226 has been started. 2023-11-12T09:29:37,374 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [test-1699781326335, ns2:test-16997813263351] 2023-11-12T09:29:37,376 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(76): StartCode 1699781321500for backupID backup_1699781375226 2023-11-12T09:29:37,376 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:29:37,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:37,377 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:37,377 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:37,379 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:37,379 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:37,379 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:37,381 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,381 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:37,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:37,381 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:37,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:37,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:37,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:37,382 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:37,382 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:37,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:37,382 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:37,382 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:37,383 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:37,383 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:37,383 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:37,385 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:37,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:37,386 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:37,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:37,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:37,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:37,388 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:37,388 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3eb6cc43[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:37,388 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:37,388 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,389 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,389 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:37,390 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:37,390 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:37,390 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,390 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781365098 highest: 1699781365117 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,390 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0:(num 1699781365098) roll requested 2023-11-12T09:29:37,401 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390, exclude list is [], retry=0 2023-11-12T09:29:37,405 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:37,407 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 with entries=13, filesize=3.56 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 2023-11-12T09:29:37,407 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:37,407 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 is not closed yet, will try archiving it next time 2023-11-12T09:29:37,407 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1:(num 1699781365117) roll requested 2023-11-12T09:29:37,409 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 not finished, retry = 0 2023-11-12T09:29:37,418 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407, exclude list is [], retry=0 2023-11-12T09:29:37,421 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:37,423 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 2023-11-12T09:29:37,423 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:37,423 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 is not closed yet, will try archiving it next time 2023-11-12T09:29:37,423 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta:.meta(num 1699781365134) roll requested 2023-11-12T09:29:37,425 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 not finished, retry = 0 2023-11-12T09:29:37,434 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781377423.meta, exclude list is [], retry=0 2023-11-12T09:29:37,437 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:37,439 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781377423.meta 2023-11-12T09:29:37,439 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:37,439 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:37,450 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-11-12T09:29:37,450 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781377390 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,458 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:37,458 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37291,1699781316827' in zk 2023-11-12T09:29:37,462 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:37,462 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:37,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,462 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:37,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:37,463 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:37,463 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:37,463 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:37,464 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,465 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,465 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:37,465 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37291,1699781316827': 2023-11-12T09:29:37,466 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:37,466 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:37,466 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:37,467 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:37,467 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:37,467 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:37,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:37,469 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:37,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,469 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:37,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:37,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:37,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:37,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:37,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,473 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:37,476 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:37,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:37,476 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:37,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:37,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:37,477 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:37,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:37,526 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 2023-11-12T09:29:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:37,580 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37291=1699781321500} newestTimestamps: {jenkins-hbase3.apache.org:37291=1699781365117} 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781377423.meta 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:37,583 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 2023-11-12T09:29:37,993 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997813263351 target: hdfs://localhost:36805/backupUT/backup_1699781375226/ns2/test-16997813263351 descriptor: 'ns2:test-16997813263351', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:37,993 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:29:37,995 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997813263351 2023-11-12T09:29:38,262 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:38,401 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997813263351 2023-11-12T09:29:38,814 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781326335 target: hdfs://localhost:36805/backupUT/backup_1699781375226/default/test-1699781326335 descriptor: 'test-1699781326335', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:38,814 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:29:38,816 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(152): Starting to write region info for table test-1699781326335 2023-11-12T09:29:39,221 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(159): Finished writing region info for table test-1699781326335 2023-11-12T09:29:39,228 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:36805/backupUT/.tmp/backup_1699781375226 from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 2023-11-12T09:29:39,241 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2131f0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:39,242 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:39,243 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42750, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:39,245 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:29:39,245 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781326335 2023-11-12T09:29:39,249 DEBUG [hconnection-0x329b1c24-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:39,250 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:39,252 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781326335: test-1699781326335; 2023-11-12T09:29:39,252 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997813263351 2023-11-12T09:29:39,253 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997813263351: ns2:test-16997813263351; 2023-11-12T09:29:39,253 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:29:39,254 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/hbase-staging/partitions_a7454dee-8cc8-4646-b4c8-95082718c219 2023-11-12T09:29:39,660 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:39,661 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:39,661 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:39,662 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,246 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 9daf39e46084aad75149b683e2ae1693 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:29:40,625 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-991055127277801752.jar 2023-11-12T09:29:40,625 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,625 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,626 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,626 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,627 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,627 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,627 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,628 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,628 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,629 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:40,630 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:40,630 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:40,630 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:40,631 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:40,631 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:40,632 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:40,632 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:40,633 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:40,633 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:40,633 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:40,634 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:40,634 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,635 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,635 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,636 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,636 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,637 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:40,637 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:40,637 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781326335,ns2:test-16997813263351 2023-11-12T09:29:40,638 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:40,639 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:41,598 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-2750936465909194733.jar 2023-11-12T09:29:41,618 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 for WAL files 2023-11-12T09:29:41,619 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:41,619 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 for WAL files 2023-11-12T09:29:41,620 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 2023-11-12T09:29:41,620 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 for WAL files 2023-11-12T09:29:41,621 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:41,621 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 for WAL files 2023-11-12T09:29:41,622 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 2023-11-12T09:29:41,696 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781381660/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:41,696 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781381660/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:41,768 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 (-9223372036854775808:9223372036854775807) length:18860 2023-11-12T09:29:41,775 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:41,776 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:41,801 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 (-9223372036854775808:9223372036854775807) length:3657 2023-11-12T09:29:41,804 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:41,804 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:41,826 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 (-9223372036854775808:9223372036854775807) length:1198 2023-11-12T09:29:41,829 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:41,829 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:41,870 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 (-9223372036854775808:9223372036854775807) length:93 2023-11-12T09:29:41,872 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:41,872 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:41,893 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a24fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:41,898 DEBUG [hconnection-0x3a2f95d0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:41,899 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36032, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:41,901 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:41,902 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-12T09:29:41,902 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:41,907 DEBUG [pool-249-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/backupUT/.tmp/backup_1699781375226/_temporary/0/_temporary/attempt_local2070186189_0005_r_000000_0/ns2/test-16997813263351/f/6aac25fb40ee4b1da852e7890ec88408 is 38, key is row-t20/f:q1/1699781353933/Put/seqid=0 2023-11-12T09:29:41,908 WARN [Thread-1080 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37291] Specified favored nodes: [jenkins-hbase3.apache.org:37291] 2023-11-12T09:29:41,951 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c3753e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:41,953 DEBUG [hconnection-0x1eb7d58a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:41,955 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:41,957 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:41,958 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1699781326335-1-0] 2023-11-12T09:29:41,958 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:41,968 DEBUG [pool-249-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/backupUT/.tmp/backup_1699781375226/_temporary/0/_temporary/attempt_local2070186189_0005_r_000001_0/default/test-1699781326335/f/26c1b263bd87465bbb4831d4262ff624 is 59, key is row-test-1699781326335-1-10/f:q1/1699781353824/Put/seqid=0 2023-11-12T09:29:41,969 WARN [Thread-1090 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37291] Specified favored nodes: [jenkins-hbase3.apache.org:37291] 2023-11-12T09:29:42,757 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:36805/backupUT 2023-11-12T09:29:42,757 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781375226 2023-11-12T09:29:42,757 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:29:42,762 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:36805/backupUT/.tmp/backup_1699781375226, hdfs://localhost:36805/backupUT] 2023-11-12T09:29:43,382 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:29:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781375226 2023-11-12T09:29:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16005 bytes copied." 2023-11-12T09:29:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local315237333_0006 completed: true true 2023-11-12T09:29:43,387 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4449500 FILE: Number of bytes written=7507436 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=299764 HDFS: Number of bytes written=2422565 HDFS: Number of read operations=1017 HDFS: Number of large read operations=0 HDFS: Number of write operations=326 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1660420096 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16005 Bytes Expected=16005 Files Copied=3 DIR_COPY=7 2023-11-12T09:29:43,387 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:36805/backupUT/.tmp/backup_1699781375226 to hdfs://localhost:36805/backupUT finished. 2023-11-12T09:29:43,392 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:29:43,392 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(240): BackupId=backup_1699781375226,startts=1699781377371,failedts=1699781383392,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_3 in testing 2023-11-12T09:29:43,393 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:29:43,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable backup:system 2023-11-12T09:29:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-12T09:29:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:29:43,400 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781383400"}]},"ts":"1699781383400"} 2023-11-12T09:29:43,401 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-12T09:29:43,403 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-12T09:29:43,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN}] 2023-11-12T09:29:43,406 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN 2023-11-12T09:29:43,407 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:43,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:29:43,560 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:43,560 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(119): Close 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:43,560 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1668): Closing 2bd6ff45da8970c8169b00517ffa7e18, disabling compactions & flushes 2023-11-12T09:29:43,561 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1690): Closing region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:43,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:43,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. after waiting 0 ms 2023-11-12T09:29:43,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:43,561 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=872 B heapSize=1.71 KB 2023-11-12T09:29:43,576 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/2e14e4a142f94bdb9486f445879e6308 is 171, key is trslm:hdfs://localhost:36805/backupUT\x00ns2:test-16997813263351/meta:log-roll-map/1699781383391/Put/seqid=0 2023-11-12T09:29:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:29:43,981 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=444 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/2e14e4a142f94bdb9486f445879e6308 2023-11-12T09:29:43,990 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/01c3831be2f0411d92c174960c64838f is 218, key is session:backup_1699781375226/session:context/1699781383383/Put/seqid=0 2023-11-12T09:29:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:29:44,394 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/01c3831be2f0411d92c174960c64838f 2023-11-12T09:29:44,400 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/2e14e4a142f94bdb9486f445879e6308 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/2e14e4a142f94bdb9486f445879e6308 2023-11-12T09:29:44,405 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/2e14e4a142f94bdb9486f445879e6308, entries=3, sequenceid=56, filesize=5.6 K 2023-11-12T09:29:44,406 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/01c3831be2f0411d92c174960c64838f as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/01c3831be2f0411d92c174960c64838f 2023-11-12T09:29:44,412 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/01c3831be2f0411d92c174960c64838f, entries=1, sequenceid=56, filesize=5.2 K 2023-11-12T09:29:44,413 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(3022): Finished flush of dataSize ~872 B/872, heapSize ~1.68 KB/1720, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 852ms, sequenceid=56, compaction requested=true 2023-11-12T09:29:44,422 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8] to archive 2023-11-12T09:29:44,423 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-11-12T09:29:44,427 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/121c01cac3884fb7bafca1a28bc15718 2023-11-12T09:29:44,428 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/2b2d2630865e4cd189ee99eefe559197 2023-11-12T09:29:44,430 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/47fb20b82eee4f119ab997125e5b50b8 2023-11-12T09:29:44,468 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=42 2023-11-12T09:29:44,469 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:44,469 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1909): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:44,469 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1622): Region close journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:44,471 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(163): Closed 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:44,471 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSED 2023-11-12T09:29:44,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=68, resume processing ppid=67 2023-11-12T09:29:44,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, ppid=67, state=SUCCESS; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 1.0650 sec 2023-11-12T09:29:44,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-11-12T09:29:44,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN in 1.0710 sec 2023-11-12T09:29:44,477 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781384477"}]},"ts":"1699781384477"} 2023-11-12T09:29:44,478 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-12T09:29:44,480 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-12T09:29:44,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0830 sec 2023-11-12T09:29:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:29:44,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781384512 (current time:1699781384512). 2023-11-12T09:29:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1699781384511 VERSION not specified, setting to 2 2023-11-12T09:29:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a68ae73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:44,523 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36058, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=69, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-11-12T09:29:44,529 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-12T09:29:44,529 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:44,532 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781384511 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:44,534 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-12T09:29:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-12T09:29:44,941 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-12T09:29:44,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-12T09:29:44,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-12T09:29:44,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:44,944 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/2e14e4a142f94bdb9486f445879e6308, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:44,944 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/2e14e4a142f94bdb9486f445879e6308 2023-11-12T09:29:44,944 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:44,944 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:44,946 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/01c3831be2f0411d92c174960c64838f, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963] hfiles 2023-11-12T09:29:44,946 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/01c3831be2f0411d92c174960c64838f 2023-11-12T09:29:44,946 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 2023-11-12T09:29:44,946 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:29:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-12T09:29:45,356 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-12T09:29:45,357 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:45,358 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:45,358 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1699781384511 2023-11-12T09:29:45,358 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781384511 2023-11-12T09:29:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-12T09:29:45,771 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:45,779 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:45,780 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781384511 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1699781384511 2023-11-12T09:29:45,781 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:45,781 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-11-12T09:29:45,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781384511 table=backup:system type=FLUSH ttl=0 } in 1.2560 sec 2023-11-12T09:29:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-12T09:29:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781375249 type: FLUSH version: 2 ttl: 0 ) 2023-11-12T09:29:46,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-12T09:29:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:29:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:29:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:29:47,058 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-12T09:29:47,058 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-11-12T09:29:47,059 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:47,062 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781375249 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:47,062 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system 2023-11-12T09:29:47,064 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-12T09:29:47,064 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,070 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/2e14e4a142f94bdb9486f445879e6308 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/2e14e4a142f94bdb9486f445879e6308 2023-11-12T09:29:47,070 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:47,073 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/01c3831be2f0411d92c174960c64838f to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/01c3831be2f0411d92c174960c64838f 2023-11-12T09:29:47,074 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:47,074 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781375249 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:47,075 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781387075"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:47,077 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:47,077 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:47,077 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781387076"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781387076"}]},"ts":"1699781387076"} 2023-11-12T09:29:47,078 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:47,078 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-12T09:29:47,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:47,078 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-12T09:29:47,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781375249 type: FLUSH version: 2 ttl: 0 ) in 436 msec 2023-11-12T09:29:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:29:47,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699781384511" 2023-11-12T09:29:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1699781384511 2023-11-12T09:29:47,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$12(2662): Client=jenkins//172.31.12.81 enable backup:system 2023-11-12T09:29:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-12T09:29:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-12T09:29:47,258 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-12T09:29:47,258 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781387258"}]},"ts":"1699781387258"} 2023-11-12T09:29:47,259 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:29:47,261 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-12T09:29:47,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN}] 2023-11-12T09:29:47,263 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN 2023-11-12T09:29:47,264 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-12T09:29:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-12T09:29:47,415 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:47,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-12T09:29:47,568 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:47,572 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(130): Open backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:47,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7254): Opening region: {ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:47,573 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:47,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(888): Instantiated backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:47,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7296): checking encryption for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7299): checking classloading for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,575 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,575 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName meta 2023-11-12T09:29:47,575 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:47,584 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:47,584 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:47,584 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,585 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName session 2023-11-12T09:29:47,585 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:47,591 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 2023-11-12T09:29:47,596 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:29:47,596 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:47,597 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,598 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,600 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:29:47,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1076): writing seq id for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:47,602 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): Opened 2bd6ff45da8970c8169b00517ffa7e18; next sequenceid=60; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11085945280, jitterRate=0.032459110021591187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:29:47,603 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(992): Region open journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:47,604 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., pid=73, masterSystemTime=1699781387568 2023-11-12T09:29:47,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:47,605 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(158): Opened backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:47,606 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPEN, openSeqNum=60, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:47,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-11-12T09:29:47,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 190 msec 2023-11-12T09:29:47,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=72, resume processing ppid=71 2023-11-12T09:29:47,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, ppid=71, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN in 347 msec 2023-11-12T09:29:47,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781387610"}]},"ts":"1699781387610"} 2023-11-12T09:29:47,611 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:29:47,613 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-12T09:29:47,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; EnableTableProcedure table=backup:system in 359 msec 2023-11-12T09:29:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-12T09:29:47,860 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-12T09:29:47,861 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:47,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:47,868 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:47,868 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-11-12T09:29:47,869 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:36805/backupUT/backup_1699781375226/ns2/test-16997813263351 done. 2023-11-12T09:29:47,869 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(208): hdfs://localhost:36805/backupUT/backup_1699781375226/ns2 is empty, remove it. 2023-11-12T09:29:47,870 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:36805/backupUT/backup_1699781375226/default/test-1699781326335 done. 2023-11-12T09:29:47,870 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(208): hdfs://localhost:36805/backupUT/backup_1699781375226/default is empty, remove it. 2023-11-12T09:29:47,872 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:47,878 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(246): Backup backup_1699781375226 failed. Backup session finished. Status: FAILURE 2023-11-12T09:29:47,878 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:47,879 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] ... 41 more 2023-11-12T09:29:47,884 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 4 2023-11-12T09:29:47,901 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe57cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:47,902 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:47,903 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:47,906 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:47,907 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57126, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:47,954 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:47,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781387958 (current time:1699781387958). 2023-11-12T09:29:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1117bb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:47,967 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=74, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-11-12T09:29:47,971 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-12T09:29:47,971 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:47,974 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:47,976 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-12T09:29:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-12T09:29:48,383 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:48,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18}] 2023-11-12T09:29:48,385 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:48,536 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:48,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37291 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=75 2023-11-12T09:29:48,537 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:48,538 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-11-12T09:29:48,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/7535e64c6bed460babf7cc7a4b24e4b3 is 256, key is session:backup_1699781375226/session:context/1699781387871/Put/seqid=0 2023-11-12T09:29:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-12T09:29:48,960 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/7535e64c6bed460babf7cc7a4b24e4b3 2023-11-12T09:29:48,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/7535e64c6bed460babf7cc7a4b24e4b3 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3 2023-11-12T09:29:48,971 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3, entries=2, sequenceid=65, filesize=5.2 K 2023-11-12T09:29:48,972 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 434ms, sequenceid=65, compaction requested=true 2023-11-12T09:29:48,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2520): Flush status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. for snapshot_backup_system completed. 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb for snapshot=snapshot_backup_system 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3] hfiles 2023-11-12T09:29:48,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 for snapshot=snapshot_backup_system 2023-11-12T09:29:48,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 for snapshot=snapshot_backup_system 2023-11-12T09:29:48,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3 for snapshot=snapshot_backup_system 2023-11-12T09:29:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-12T09:29:49,379 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:49,379 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=75 2023-11-12T09:29:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster(4026): Remote procedure done, pid=75 2023-11-12T09:29:49,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:49,381 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:49,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-11-12T09:29:49,383 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:49,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; SnapshotRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18 in 999 msec 2023-11-12T09:29:49,384 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:49,384 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:49,384 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:49,385 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:49,800 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:49,807 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:49,808 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:49,809 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:49,809 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-11-12T09:29:49,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8410 sec 2023-11-12T09:29:50,014 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-12T09:29:50,077 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(115): Backup backup_1699781387932 started at 1699781390077. 2023-11-12T09:29:50,078 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(119): Backup session backup_1699781387932 has been started. 2023-11-12T09:29:50,080 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [test-1699781326335, ns2:test-16997813263351] 2023-11-12T09:29:50,082 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(76): StartCode 1699781321500for backupID backup_1699781387932 2023-11-12T09:29:50,082 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:29:50,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:50,083 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:50,083 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:50,084 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:50,084 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:50,085 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:50,086 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,086 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:50,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:50,087 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:50,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:50,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:50,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:50,088 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:50,088 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:50,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:50,088 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:50,088 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:50,089 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:50,089 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:50,089 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:50,096 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:50,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:50,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:50,096 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:50,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:50,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:50,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:50,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:50,098 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37291,1699781316827' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:50,098 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@223e4f41[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:50,098 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:50,098 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:50,100 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:50,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:50,100 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:50,100 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:50,100 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:50,100 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,100 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781377390 highest: 1699781377407 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,100 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0:(num 1699781377390) roll requested 2023-11-12T09:29:50,112 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781390100, exclude list is [], retry=0 2023-11-12T09:29:50,117 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:50,119 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 with entries=13, filesize=3.55 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781390100 2023-11-12T09:29:50,119 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:50,119 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 is not closed yet, will try archiving it next time 2023-11-12T09:29:50,119 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1:(num 1699781377407) roll requested 2023-11-12T09:29:50,119 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 2023-11-12T09:29:50,132 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781390119, exclude list is [], retry=0 2023-11-12T09:29:50,135 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:50,137 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781390119 2023-11-12T09:29:50,137 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:50,137 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 is not closed yet, will try archiving it next time 2023-11-12T09:29:50,137 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta:.meta(num 1699781377423) roll requested 2023-11-12T09:29:50,139 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 2023-11-12T09:29:50,151 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781390137.meta, exclude list is [], retry=0 2023-11-12T09:29:50,154 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK] 2023-11-12T09:29:50,156 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781377423.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781390137.meta 2023-11-12T09:29:50,156 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36527,DS-3e2ede94-3d77-4401-8f33-81278e38f079,DISK]] 2023-11-12T09:29:50,156 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781377423.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:50,160 DEBUG [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-11-12T09:29:50,160 INFO [rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781390100 on jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,168 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:50,168 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37291,1699781316827' in zk 2023-11-12T09:29:50,171 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:50,171 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:50,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,171 DEBUG [member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:50,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:50,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:50,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:50,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:50,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:50,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:50,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:50,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,175 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37291,1699781316827': 2023-11-12T09:29:50,175 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37291,1699781316827' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:50,175 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:50,175 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:50,175 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:50,175 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:50,175 INFO [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:50,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:50,177 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:50,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:50,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:50,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:50,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:50,178 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:50,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:50,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:50,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:50,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:50,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:50,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:50,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:50,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:50,190 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:50,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:50,191 DEBUG [(jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:50,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:50,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:50,192 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:50,192 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:50,294 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37291=1699781321500} newestTimestamps: {jenkins-hbase3.apache.org:37291=1699781377407} 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781319976.meta 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781342867.meta 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781365134.meta 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781377423.meta 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta.1699781390137.meta 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781390100 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:50,296 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781390119 2023-11-12T09:29:50,707 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997813263351 target: hdfs://localhost:36805/backupUT/backup_1699781387932/ns2/test-16997813263351 descriptor: 'ns2:test-16997813263351', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:50,707 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:29:50,709 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997813263351 2023-11-12T09:29:51,115 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997813263351 2023-11-12T09:29:51,525 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781326335 target: hdfs://localhost:36805/backupUT/backup_1699781387932/default/test-1699781326335 descriptor: 'test-1699781326335', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:51,525 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:29:51,527 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(152): Starting to write region info for table test-1699781326335 2023-11-12T09:29:51,933 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.BackupUtils(159): Finished writing region info for table test-1699781326335 2023-11-12T09:29:51,939 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:36805/backupUT/.tmp/backup_1699781387932 from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117;hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 2023-11-12T09:29:51,946 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283399cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:51,947 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:51,949 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55088, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:51,950 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:29:51,950 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781326335 2023-11-12T09:29:51,955 DEBUG [hconnection-0x13bf0608-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:51,956 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:51,957 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781326335: test-1699781326335; 2023-11-12T09:29:51,957 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997813263351 2023-11-12T09:29:51,959 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997813263351: ns2:test-16997813263351; 2023-11-12T09:29:51,959 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:29:51,959 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/hbase-staging/partitions_4564b3b2-4d43-4e46-8166-b8111240f1f5 2023-11-12T09:29:52,367 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:52,367 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:52,368 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:52,368 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,414 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-4653731916943205443.jar 2023-11-12T09:29:53,415 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,416 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,416 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,417 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,418 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,418 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,419 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,419 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,419 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,420 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:53,421 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:53,421 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:53,421 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:53,422 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:53,422 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:53,423 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:53,423 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:53,424 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:53,424 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:53,425 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:53,425 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:53,426 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,427 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,427 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,428 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,428 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,429 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:53,430 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:53,430 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781326335,ns2:test-16997813263351 2023-11-12T09:29:53,430 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:53,431 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:54,452 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop-7902630730907157296.jar 2023-11-12T09:29:54,474 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 for WAL files 2023-11-12T09:29:54,476 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 2023-11-12T09:29:54,476 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 for WAL files 2023-11-12T09:29:54,477 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 2023-11-12T09:29:54,477 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 for WAL files 2023-11-12T09:29:54,478 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 2023-11-12T09:29:54,478 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 for WAL files 2023-11-12T09:29:54,478 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 2023-11-12T09:29:54,478 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 for WAL files 2023-11-12T09:29:54,479 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 2023-11-12T09:29:54,479 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 for WAL files 2023-11-12T09:29:54,480 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 2023-11-12T09:29:54,557 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781394521/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:54,557 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/mapreduce.cluster.local.dir/1699781394521/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:54,612 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781342828 (-9223372036854775808:9223372036854775807) length:18860 2023-11-12T09:29:54,618 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:54,618 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:54,676 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781365098 (-9223372036854775808:9223372036854775807) length:3657 2023-11-12T09:29:54,679 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:54,679 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:54,701 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0.1699781377390 (-9223372036854775808:9223372036854775807) length:3644 2023-11-12T09:29:54,704 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:54,704 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:54,726 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781342848 (-9223372036854775808:9223372036854775807) length:1198 2023-11-12T09:29:54,730 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:54,730 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:54,752 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781365117 (-9223372036854775808:9223372036854775807) length:93 2023-11-12T09:29:54,754 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:54,754 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:54,776 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 for hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781377407 (-9223372036854775808:9223372036854775807) length:93 2023-11-12T09:29:54,779 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:29:54,779 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:29:54,798 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406aeeb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:54,799 DEBUG [hconnection-0x70bbf1f0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:54,801 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:54,802 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:54,804 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-12T09:29:54,804 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:54,808 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/backupUT/.tmp/backup_1699781387932/_temporary/0/_temporary/attempt_local447144393_0007_r_000000_0/ns2/test-16997813263351/f/d020c64313f645249ffd3fa18d2a5c38 is 38, key is row-t20/f:q1/1699781353933/Put/seqid=0 2023-11-12T09:29:54,809 WARN [Thread-1242 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37291] Specified favored nodes: [jenkins-hbase3.apache.org:37291] 2023-11-12T09:29:54,882 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:29:54,882 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:29:55,238 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c29a3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:55,240 DEBUG [hconnection-0x39ff6b55-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:55,243 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37812, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:55,245 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:55,249 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1699781326335-1-0] 2023-11-12T09:29:55,249 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:55,258 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/backupUT/.tmp/backup_1699781387932/_temporary/0/_temporary/attempt_local447144393_0007_r_000001_0/default/test-1699781326335/f/64f1459eaf5343448fd7f7adaf85bb31 is 59, key is row-test-1699781326335-1-10/f:q1/1699781353824/Put/seqid=0 2023-11-12T09:29:55,260 WARN [Thread-1252 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37291] Specified favored nodes: [jenkins-hbase3.apache.org:37291] 2023-11-12T09:29:56,596 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:36805/backupUT 2023-11-12T09:29:56,596 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781387932 2023-11-12T09:29:56,596 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:29:56,603 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:36805/backupUT/.tmp/backup_1699781387932, hdfs://localhost:36805/backupUT] 2023-11-12T09:29:57,229 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:29:57,231 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781387932 2023-11-12T09:29:57,231 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-11-12T09:29:57,232 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1588877219_0008 completed: true true 2023-11-12T09:29:57,234 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5110550 FILE: Number of bytes written=9935108 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=399218 HDFS: Number of bytes written=2475173 HDFS: Number of read operations=1321 HDFS: Number of large read operations=0 HDFS: Number of write operations=413 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1639448576 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-11-12T09:29:57,234 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:36805/backupUT/.tmp/backup_1699781387932 to hdfs://localhost:36805/backupUT finished. 2023-11-12T09:29:57,245 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:29:57,245 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:29:57,246 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(240): BackupId=backup_1699781387932,startts=1699781390077,failedts=1699781397246,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_4 in testing 2023-11-12T09:29:57,246 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:29:57,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable backup:system 2023-11-12T09:29:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-12T09:29:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-12T09:29:57,255 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781397255"}]},"ts":"1699781397255"} 2023-11-12T09:29:57,256 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-12T09:29:57,258 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-12T09:29:57,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN}] 2023-11-12T09:29:57,260 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN 2023-11-12T09:29:57,261 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:57,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-12T09:29:57,413 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:57,413 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(119): Close 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:57,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1668): Closing 2bd6ff45da8970c8169b00517ffa7e18, disabling compactions & flushes 2023-11-12T09:29:57,414 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1690): Closing region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:57,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:57,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. after waiting 0 ms 2023-11-12T09:29:57,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:57,414 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=959 B heapSize=1.87 KB 2023-11-12T09:29:57,427 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/8ffff21a4678449287cbc02532848d08 is 171, key is trslm:hdfs://localhost:36805/backupUT\x00ns2:test-16997813263351/meta:log-roll-map/1699781397238/Put/seqid=0 2023-11-12T09:29:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-12T09:29:57,832 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/8ffff21a4678449287cbc02532848d08 2023-11-12T09:29:57,841 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/5c84fd2dac024211a316e483d2c3ca8d is 218, key is session:backup_1699781387932/session:context/1699781397230/Put/seqid=0 2023-11-12T09:29:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-12T09:29:58,246 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/5c84fd2dac024211a316e483d2c3ca8d 2023-11-12T09:29:58,252 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/meta/8ffff21a4678449287cbc02532848d08 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/8ffff21a4678449287cbc02532848d08 2023-11-12T09:29:58,257 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/8ffff21a4678449287cbc02532848d08, entries=4, sequenceid=73, filesize=5.7 K 2023-11-12T09:29:58,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/5c84fd2dac024211a316e483d2c3ca8d as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/5c84fd2dac024211a316e483d2c3ca8d 2023-11-12T09:29:58,264 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/5c84fd2dac024211a316e483d2c3ca8d, entries=1, sequenceid=73, filesize=5.2 K 2023-11-12T09:29:58,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~959 B/959, heapSize ~1.84 KB/1880, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 851ms, sequenceid=73, compaction requested=true 2023-11-12T09:29:58,275 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/recovered.edits/76.seqid, newMaxSeqId=76, maxSeqId=59 2023-11-12T09:29:58,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:58,277 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1909): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:58,277 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1622): Region close journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:58,278 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(163): Closed 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:58,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=CLOSED 2023-11-12T09:29:58,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=78, resume processing ppid=77 2023-11-12T09:29:58,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, ppid=77, state=SUCCESS; CloseRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 1.0190 sec 2023-11-12T09:29:58,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-11-12T09:29:58,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, UNASSIGN in 1.0230 sec 2023-11-12T09:29:58,284 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781398283"}]},"ts":"1699781398283"} 2023-11-12T09:29:58,285 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-12T09:29:58,286 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-12T09:29:58,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0360 sec 2023-11-12T09:29:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-12T09:29:58,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781398368 (current time:1699781398368). 2023-11-12T09:29:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1699781398367 VERSION not specified, setting to 2 2023-11-12T09:29:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5504c98a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:58,378 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37816, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-12T09:29:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:29:58,384 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:58,385 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:58,387 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot, current working dir path: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781398367 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:58,389 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:58,398 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-12T09:29:58,399 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-12T09:29:58,399 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-12T09:29:58,399 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:58,400 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/8ffff21a4678449287cbc02532848d08, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb] hfiles 2023-11-12T09:29:58,400 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/8ffff21a4678449287cbc02532848d08 2023-11-12T09:29:58,400 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:58,400 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:58,402 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/5c84fd2dac024211a316e483d2c3ca8d, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963] hfiles 2023-11-12T09:29:58,402 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/5c84fd2dac024211a316e483d2c3ca8d 2023-11-12T09:29:58,403 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3 2023-11-12T09:29:58,403 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 2023-11-12T09:29:58,403 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:29:58,408 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-12T09:29:58,408 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:58,409 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:58,409 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1699781398367 2023-11-12T09:29:58,410 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781398367 2023-11-12T09:29:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:29:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:29:58,826 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:58,834 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:58,835 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699781398367 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1699781398367 2023-11-12T09:29:58,836 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:58,837 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-12T09:29:58,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699781398367 table=backup:system type=FLUSH ttl=0 } in 457 msec 2023-11-12T09:29:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:29:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=80, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781387958 type: FLUSH version: 2 ttl: 0 ) 2023-11-12T09:29:58,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-12T09:29:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=80 2023-11-12T09:29:59,014 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-11-12T09:29:59,014 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-11-12T09:29:59,015 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:59,017 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781387958 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:59,017 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system 2023-11-12T09:29:59,019 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-12T09:29:59,019 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,025 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/8ffff21a4678449287cbc02532848d08 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/8ffff21a4678449287cbc02532848d08 2023-11-12T09:29:59,025 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:59,028 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/5c84fd2dac024211a316e483d2c3ca8d to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/5c84fd2dac024211a316e483d2c3ca8d 2023-11-12T09:29:59,028 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:59,028 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781387958 type: FLUSH version: 2 ttl: 0 2023-11-12T09:29:59,030 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781399029"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:59,032 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:59,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:59,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781399030"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781399030"}]},"ts":"1699781399030"} 2023-11-12T09:29:59,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:59,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-12T09:29:59,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:59,033 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-12T09:29:59,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699781387958 type: FLUSH version: 2 ttl: 0 ) in 37 msec 2023-11-12T09:29:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=80 2023-11-12T09:29:59,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699781398367" 2023-11-12T09:29:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1699781398367 2023-11-12T09:29:59,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.HMaster$12(2662): Client=jenkins//172.31.12.81 enable backup:system 2023-11-12T09:29:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-12T09:29:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:29:59,110 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-12T09:29:59,110 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781399110"}]},"ts":"1699781399110"} 2023-11-12T09:29:59,112 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:29:59,116 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-12T09:29:59,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN}] 2023-11-12T09:29:59,119 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN 2023-11-12T09:29:59,120 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-12T09:29:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:29:59,271 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:59,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827}] 2023-11-12T09:29:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:29:59,425 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:59,428 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(130): Open backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:59,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 2bd6ff45da8970c8169b00517ffa7e18, NAME => 'backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:59,428 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:59,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(888): Instantiated backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:59,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,430 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,431 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName meta 2023-11-12T09:29:59,431 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:59,437 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/meta/d4112b1d553941878d79afd85b34fccb 2023-11-12T09:29:59,437 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:59,437 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,438 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bd6ff45da8970c8169b00517ffa7e18 columnFamilyName session 2023-11-12T09:29:59,438 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:59,445 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3 2023-11-12T09:29:59,449 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 2023-11-12T09:29:59,481 DEBUG [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:29:59,481 INFO [StoreOpener-2bd6ff45da8970c8169b00517ffa7e18-1 {}] regionserver.HStore(324): Store=2bd6ff45da8970c8169b00517ffa7e18/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:59,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,483 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,484 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:29:59,486 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,486 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): Opened 2bd6ff45da8970c8169b00517ffa7e18; next sequenceid=77; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10583574400, jitterRate=-0.014327824115753174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:29:59,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:59,488 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., pid=83, masterSystemTime=1699781399425 2023-11-12T09:29:59,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(403): Add compact mark for store 2bd6ff45da8970c8169b00517ffa7e18:session, priority=-2147483648, current under compaction store size is 1 2023-11-12T09:29:59,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-11-12T09:29:59,492 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-11-12T09:29:59,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=2bd6ff45da8970c8169b00517ffa7e18, regionState=OPEN, openSeqNum=77, regionLocation=jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:59,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:59,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(158): Opened backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:59,494 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16834 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-11-12T09:29:59,494 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HStore(1534): 2bd6ff45da8970c8169b00517ffa7e18/session is initiating minor compaction (all files) 2023-11-12T09:29:59,494 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 2bd6ff45da8970c8169b00517ffa7e18/session in backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:29:59,495 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3] into tmpdir=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp, totalSize=16.4 K 2023-11-12T09:29:59,496 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.Compactor(223): Compacting 80629eb81b874242bb02bb184d9fd334, keycount=4, bloomtype=ROW, size=6.0 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1699781353775 2023-11-12T09:29:59,496 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.Compactor(223): Compacting bdb2a855702c40c3814f7cdbdd5eb963, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1699781375164 2023-11-12T09:29:59,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-11-12T09:29:59,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 2bd6ff45da8970c8169b00517ffa7e18, server=jenkins-hbase3.apache.org,37291,1699781316827 in 221 msec 2023-11-12T09:29:59,497 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7535e64c6bed460babf7cc7a4b24e4b3, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1699781387871 2023-11-12T09:29:59,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-11-12T09:29:59,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=2bd6ff45da8970c8169b00517ffa7e18, ASSIGN in 380 msec 2023-11-12T09:29:59,499 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781399499"}]},"ts":"1699781399499"} 2023-11-12T09:29:59,501 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:29:59,503 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-12T09:29:59,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; EnableTableProcedure table=backup:system in 398 msec 2023-11-12T09:29:59,522 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bd6ff45da8970c8169b00517ffa7e18#session#compaction#17 average throughput is 1.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-11-12T09:29:59,523 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/f6cbe0072bd448b38c030eda7d592cc2 is 459, key is session:backup_1699781336188/session:context/1699781353775/Put/seqid=0 2023-11-12T09:29:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:29:59,713 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-12T09:29:59,713 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:59,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43745 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:59,721 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:59,721 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-11-12T09:29:59,722 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:36805/backupUT/backup_1699781387932/ns2/test-16997813263351 done. 2023-11-12T09:29:59,722 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(208): hdfs://localhost:36805/backupUT/backup_1699781387932/ns2 is empty, remove it. 2023-11-12T09:29:59,723 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:36805/backupUT/backup_1699781387932/default/test-1699781326335 done. 2023-11-12T09:29:59,724 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(208): hdfs://localhost:36805/backupUT/backup_1699781387932/default is empty, remove it. 2023-11-12T09:29:59,726 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:59,730 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] impl.TableBackupClient(246): Backup backup_1699781387932 failed. Backup session finished. Status: FAILURE 2023-11-12T09:29:59,732 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:59,732 ERROR [Listener at jenkins-hbase3.apache.org/42507 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] ... 41 more 2023-11-12T09:29:59,737 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:59,760 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=602 (was 587) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_680175165_16 at /127.0.0.1:38178 [Waiting for operation #7] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,43745,1699781313745)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2054777967-172.31.12.81-1699781310530:blk_1073741940_1116, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,37291,1699781316827' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57937@0x619e6dfe java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$539/2045302963.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_680175165_16 at /127.0.0.1:38172 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/f6cbe0072bd448b38c030eda7d592cc2 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:678) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1738162312_16 at /127.0.0.1:57304 [Receiving block BP-2054777967-172.31.12.81-1699781310530:blk_1073741939_1115] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2054777967-172.31.12.81-1699781310530:blk_1073741938_1114, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:37291-shortCompactions-0 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:940) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:880) org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:863) org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:818) org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:858) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:679) org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:377) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:72) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:40) org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:396) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:64) org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1176) org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2341) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1738162312_16 at /127.0.0.1:57294 [Receiving block BP-2054777967-172.31.12.81-1699781310530:blk_1073741938_1114] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x45030a3a-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2054777967-172.31.12.81-1699781310530:blk_1073741939_1115, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37291,1699781316827)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x45030a3a-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_680175165_16 at /127.0.0.1:38144 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_680175165_16 at /127.0.0.1:57360 [Waiting for operation #32] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1738162312_16 at /127.0.0.1:57308 [Receiving block BP-2054777967-172.31.12.81-1699781310530:blk_1073741940_1116] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2e7d6f30-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_680175165_16 at /127.0.0.1:38118 [Waiting for operation #10] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57937@0x619e6dfe-SendThread(127.0.0.1:57937) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57937@0x619e6dfe-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=714 (was 696) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=120 (was 193), ProcessCount=169 (was 171), AvailableMemoryMB=5976 (was 5378) - AvailableMemoryMB LEAK? - 2023-11-12T09:29:59,761 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-11-12T09:29:59,770 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-12T09:29:59,771 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:59,771 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-12T09:29:59,771 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] util.JVMClusterUtil(257): Found active master hash=920710274, stopped=false 2023-11-12T09:29:59,771 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:29:59,774 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-12T09:29:59,775 DEBUG [Listener at jenkins-hbase3.apache.org/42507 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:59,775 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37291,1699781316827' ***** 2023-11-12T09:29:59,775 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-12T09:29:59,775 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:29:59,780 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:29:59,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-12T09:29:59,791 INFO [RS:0;jenkins-hbase3:37291 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@462a200{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:29:59,795 INFO [RS:0;jenkins-hbase3:37291 {}] server.AbstractConnector(383): Stopped ServerConnector@45201661{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:29:59,796 INFO [RS:0;jenkins-hbase3:37291 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:29:59,796 INFO [RS:0;jenkins-hbase3:37291 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4c16fe40{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:29:59,797 INFO [RS:0;jenkins-hbase3:37291 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1650a57f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.log.dir/,STOPPED} 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 7a46314c3b11bd429216bed3cf7244cb 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 4330779e284cca1387db0585407c975a 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 2bd6ff45da8970c8169b00517ffa7e18 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 355fc0ccb227f73de53d754779d91940 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:29:59,800 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 3c0632ff163ee5a4367c764a083d82ca 2023-11-12T09:29:59,801 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 6ab12e4c0c19d984646cbe9c7422a1e3 2023-11-12T09:29:59,801 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37291,1699781316827 2023-11-12T09:29:59,801 DEBUG [RS:0;jenkins-hbase3:37291 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:59,801 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-12T09:29:59,801 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-12T09:29:59,801 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7a46314c3b11bd429216bed3cf7244cb, disabling compactions & flushes 2023-11-12T09:29:59,801 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:29:59,801 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:29:59,801 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. after waiting 0 ms 2023-11-12T09:29:59,801 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:29:59,801 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7a46314c3b11bd429216bed3cf7244cb 1/1 column families, dataSize=170 B heapSize=816 B 2023-11-12T09:29:59,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/.tmp/f/45be48e70c094a6695d930e56385845d is 38, key is row-t20/f:q1/1699781353933/Put/seqid=0 2023-11-12T09:29:59,888 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-12T09:29:59,888 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-12T09:29:59,939 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/f6cbe0072bd448b38c030eda7d592cc2 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/f6cbe0072bd448b38c030eda7d592cc2 2023-11-12T09:29:59,945 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 2bd6ff45da8970c8169b00517ffa7e18/session of 2bd6ff45da8970c8169b00517ffa7e18 into f6cbe0072bd448b38c030eda7d592cc2(size=6.5 K), total size for store is 6.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-11-12T09:29:59,945 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:29:59,945 INFO [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., storeName=2bd6ff45da8970c8169b00517ffa7e18/session, priority=13, startTime=1699781399488; duration=0sec 2023-11-12T09:29:59,945 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-12T09:29:59,945 DEBUG [RS:0;jenkins-hbase3:37291-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bd6ff45da8970c8169b00517ffa7e18:session 2023-11-12T09:29:59,945 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-12T09:29:59,945 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-12T09:29:59,946 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-12T09:29:59,946 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1583): Online Regions={7a46314c3b11bd429216bed3cf7244cb=ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb., 4330779e284cca1387db0585407c975a=ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a., 2bd6ff45da8970c8169b00517ffa7e18=backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18., 355fc0ccb227f73de53d754779d91940=test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940., 1588230740=hbase:meta,,1.1588230740, 9daf39e46084aad75149b683e2ae1693=hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693., 3c0632ff163ee5a4367c764a083d82ca=backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca., 6ab12e4c0c19d984646cbe9c7422a1e3=ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.} 2023-11-12T09:29:59,947 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:29:59,947 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:29:59,947 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:29:59,947 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:29:59,947 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:29:59,947 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=18.81 KB heapSize=32.60 KB 2023-11-12T09:29:59,950 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2bd6ff45da8970c8169b00517ffa7e18, 355fc0ccb227f73de53d754779d91940, 3c0632ff163ee5a4367c764a083d82ca, 4330779e284cca1387db0585407c975a, 6ab12e4c0c19d984646cbe9c7422a1e3, 7a46314c3b11bd429216bed3cf7244cb, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:29:59,970 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/.tmp/info/99867165bc1b458087bb81e55bd4ea89 is 159, key is ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb./info:regioninfo/1699781329243/Put/seqid=0 2023-11-12T09:30:00,150 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2bd6ff45da8970c8169b00517ffa7e18, 355fc0ccb227f73de53d754779d91940, 3c0632ff163ee5a4367c764a083d82ca, 4330779e284cca1387db0585407c975a, 6ab12e4c0c19d984646cbe9c7422a1e3, 7a46314c3b11bd429216bed3cf7244cb, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:00,210 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/.tmp/f/45be48e70c094a6695d930e56385845d 2023-11-12T09:30:00,217 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/.tmp/f/45be48e70c094a6695d930e56385845d as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/45be48e70c094a6695d930e56385845d 2023-11-12T09:30:00,222 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/f/45be48e70c094a6695d930e56385845d, entries=5, sequenceid=111, filesize=5.0 K 2023-11-12T09:30:00,223 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 7a46314c3b11bd429216bed3cf7244cb in 422ms, sequenceid=111, compaction requested=false 2023-11-12T09:30:00,229 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns2/test-16997813263351/7a46314c3b11bd429216bed3cf7244cb/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2023-11-12T09:30:00,230 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:00,230 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:30:00,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7a46314c3b11bd429216bed3cf7244cb: 2023-11-12T09:30:00,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16997813263351,,1699781328055.7a46314c3b11bd429216bed3cf7244cb. 2023-11-12T09:30:00,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4330779e284cca1387db0585407c975a, disabling compactions & flushes 2023-11-12T09:30:00,231 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:30:00,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:30:00,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. after waiting 0 ms 2023-11-12T09:30:00,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:30:00,234 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns3/test-16997813263352/4330779e284cca1387db0585407c975a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:30:00,235 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:00,236 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:30:00,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4330779e284cca1387db0585407c975a: 2023-11-12T09:30:00,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16997813263352,,1699781330256.4330779e284cca1387db0585407c975a. 2023-11-12T09:30:00,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2bd6ff45da8970c8169b00517ffa7e18, disabling compactions & flushes 2023-11-12T09:30:00,236 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:30:00,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:30:00,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. after waiting 0 ms 2023-11-12T09:30:00,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:30:00,236 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2bd6ff45da8970c8169b00517ffa7e18 2/2 column families, dataSize=296 B heapSize=960 B 2023-11-12T09:30:00,243 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/e72ddfd2c3164920b9965ba18d525f4f is 256, key is session:backup_1699781387932/session:context/1699781399725/Put/seqid=0 2023-11-12T09:30:00,351 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2bd6ff45da8970c8169b00517ffa7e18, 355fc0ccb227f73de53d754779d91940, 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:00,377 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.41 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/.tmp/info/99867165bc1b458087bb81e55bd4ea89 2023-11-12T09:30:00,398 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/.tmp/table/5bdb0108b53b4ab89206e043ce3d24dd is 59, key is ns2:test-16997813263351/table:state/1699781329253/Put/seqid=0 2023-11-12T09:30:00,551 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2bd6ff45da8970c8169b00517ffa7e18, 355fc0ccb227f73de53d754779d91940, 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:00,653 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=296 B at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/e72ddfd2c3164920b9965ba18d525f4f 2023-11-12T09:30:00,659 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/.tmp/session/e72ddfd2c3164920b9965ba18d525f4f as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/e72ddfd2c3164920b9965ba18d525f4f 2023-11-12T09:30:00,663 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/e72ddfd2c3164920b9965ba18d525f4f, entries=2, sequenceid=82, filesize=5.2 K 2023-11-12T09:30:00,664 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~296 B/296, heapSize ~688 B/688, currentSize=0 B/0 for 2bd6ff45da8970c8169b00517ffa7e18 in 428ms, sequenceid=82, compaction requested=false 2023-11-12T09:30:00,672 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3] to archive 2023-11-12T09:30:00,673 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-11-12T09:30:00,674 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/80629eb81b874242bb02bb184d9fd334 2023-11-12T09:30:00,676 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/bdb2a855702c40c3814f7cdbdd5eb963 2023-11-12T09:30:00,678 DEBUG [StoreCloser-backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3 to hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/archive/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/session/7535e64c6bed460babf7cc7a4b24e4b3 2023-11-12T09:30:00,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system/2bd6ff45da8970c8169b00517ffa7e18/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=76 2023-11-12T09:30:00,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:00,687 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:30:00,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2bd6ff45da8970c8169b00517ffa7e18: 2023-11-12T09:30:00,688 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1699781336337.2bd6ff45da8970c8169b00517ffa7e18. 2023-11-12T09:30:00,688 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 355fc0ccb227f73de53d754779d91940, disabling compactions & flushes 2023-11-12T09:30:00,688 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:30:00,688 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:30:00,688 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. after waiting 0 ms 2023-11-12T09:30:00,688 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:30:00,688 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 355fc0ccb227f73de53d754779d91940 2/2 column families, dataSize=5.30 KB heapSize=12.88 KB 2023-11-12T09:30:00,702 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f/ec00dcdf5d574174b1981d01530d8df8 is 59, key is row-test-1699781326335-1-10/f:q1/1699781353824/Put/seqid=0 2023-11-12T09:30:00,751 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 355fc0ccb227f73de53d754779d91940, 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:00,803 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/.tmp/table/5bdb0108b53b4ab89206e043ce3d24dd 2023-11-12T09:30:00,809 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/.tmp/info/99867165bc1b458087bb81e55bd4ea89 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/info/99867165bc1b458087bb81e55bd4ea89 2023-11-12T09:30:00,814 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/info/99867165bc1b458087bb81e55bd4ea89, entries=79, sequenceid=83, filesize=15.1 K 2023-11-12T09:30:00,815 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/.tmp/table/5bdb0108b53b4ab89206e043ce3d24dd as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/table/5bdb0108b53b4ab89206e043ce3d24dd 2023-11-12T09:30:00,820 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/table/5bdb0108b53b4ab89206e043ce3d24dd, entries=15, sequenceid=83, filesize=5.8 K 2023-11-12T09:30:00,820 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~18.81 KB/19263, heapSize ~32.32 KB/33096, currentSize=0 B/0 for 1588230740 in 873ms, sequenceid=83, compaction requested=false 2023-11-12T09:30:00,829 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/meta/1588230740/recovered.edits/86.seqid, newMaxSeqId=86, maxSeqId=1 2023-11-12T09:30:00,830 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:00,830 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:30:00,830 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:30:00,830 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:30:00,831 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-12T09:30:00,952 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-11-12T09:30:00,952 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1583): Online Regions={355fc0ccb227f73de53d754779d91940=test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940., 9daf39e46084aad75149b683e2ae1693=hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693., 3c0632ff163ee5a4367c764a083d82ca=backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca., 6ab12e4c0c19d984646cbe9c7422a1e3=ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3.} 2023-11-12T09:30:00,952 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 355fc0ccb227f73de53d754779d91940, 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:01,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f/ec00dcdf5d574174b1981d01530d8df8 2023-11-12T09:30:01,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/.tmp/f/ec00dcdf5d574174b1981d01530d8df8 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/ec00dcdf5d574174b1981d01530d8df8 2023-11-12T09:30:01,124 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/f/ec00dcdf5d574174b1981d01530d8df8, entries=99, sequenceid=217, filesize=10.5 K 2023-11-12T09:30:01,124 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.30 KB/5425, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 355fc0ccb227f73de53d754779d91940 in 436ms, sequenceid=217, compaction requested=false 2023-11-12T09:30:01,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/default/test-1699781326335/355fc0ccb227f73de53d754779d91940/recovered.edits/220.seqid, newMaxSeqId=220, maxSeqId=106 2023-11-12T09:30:01,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:01,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:30:01,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 355fc0ccb227f73de53d754779d91940: 2023-11-12T09:30:01,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1699781326335,,1699781326817.355fc0ccb227f73de53d754779d91940. 2023-11-12T09:30:01,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9daf39e46084aad75149b683e2ae1693, disabling compactions & flushes 2023-11-12T09:30:01,133 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:30:01,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:30:01,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. after waiting 0 ms 2023-11-12T09:30:01,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:30:01,133 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9daf39e46084aad75149b683e2ae1693 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-12T09:30:01,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/.tmp/info/3161522f4e38401eb6292400ebfa8bf2 is 45, key is default/info:d/1699781321641/Put/seqid=0 2023-11-12T09:30:01,152 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:01,352 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:01,550 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/.tmp/info/3161522f4e38401eb6292400ebfa8bf2 2023-11-12T09:30:01,552 DEBUG [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1609): Waiting on 3c0632ff163ee5a4367c764a083d82ca, 6ab12e4c0c19d984646cbe9c7422a1e3, 9daf39e46084aad75149b683e2ae1693 2023-11-12T09:30:01,557 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/.tmp/info/3161522f4e38401eb6292400ebfa8bf2 as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/info/3161522f4e38401eb6292400ebfa8bf2 2023-11-12T09:30:01,562 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/info/3161522f4e38401eb6292400ebfa8bf2, entries=7, sequenceid=11, filesize=5.1 K 2023-11-12T09:30:01,563 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 9daf39e46084aad75149b683e2ae1693 in 430ms, sequenceid=11, compaction requested=false 2023-11-12T09:30:01,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/hbase/namespace/9daf39e46084aad75149b683e2ae1693/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-12T09:30:01,576 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:01,577 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:30:01,577 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9daf39e46084aad75149b683e2ae1693: 2023-11-12T09:30:01,578 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1699781320257.9daf39e46084aad75149b683e2ae1693. 2023-11-12T09:30:01,578 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3c0632ff163ee5a4367c764a083d82ca, disabling compactions & flushes 2023-11-12T09:30:01,578 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:30:01,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:30:01,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. after waiting 0 ms 2023-11-12T09:30:01,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:30:01,584 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/backup/system_bulk/3c0632ff163ee5a4367c764a083d82ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:30:01,585 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:01,587 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:30:01,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3c0632ff163ee5a4367c764a083d82ca: 2023-11-12T09:30:01,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1699781338467.3c0632ff163ee5a4367c764a083d82ca. 2023-11-12T09:30:01,588 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 6ab12e4c0c19d984646cbe9c7422a1e3, disabling compactions & flushes 2023-11-12T09:30:01,588 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:30:01,588 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:30:01,588 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. after waiting 0 ms 2023-11-12T09:30:01,588 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:30:01,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/data/ns4/test-16997813263353/6ab12e4c0c19d984646cbe9c7422a1e3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:30:01,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:30:01,601 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:30:01,601 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 6ab12e4c0c19d984646cbe9c7422a1e3: 2023-11-12T09:30:01,601 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16997813263353,,1699781332381.6ab12e4c0c19d984646cbe9c7422a1e3. 2023-11-12T09:30:01,753 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37291,1699781316827; all regions closed. 2023-11-12T09:30:01,768 DEBUG [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(1062): Moved 5 WAL file(s) to /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs 2023-11-12T09:30:01,768 INFO [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.meta:.meta(num 1699781390137) 2023-11-12T09:30:01,771 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/WALs/jenkins-hbase3.apache.org,37291,1699781316827/jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1.1699781390119 not finished, retry = 0 2023-11-12T09:30:01,876 DEBUG [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs 2023-11-12T09:30:01,876 INFO [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-1:(num 1699781390119) 2023-11-12T09:30:01,884 DEBUG [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/oldWALs 2023-11-12T09:30:01,884 INFO [RS:0;jenkins-hbase3:37291 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37291%2C1699781316827.jenkins-hbase3.apache.org%2C37291%2C1699781316827.regiongroup-0:(num 1699781390100) 2023-11-12T09:30:01,884 DEBUG [RS:0;jenkins-hbase3:37291 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:01,884 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:30:01,885 INFO [RS:0;jenkins-hbase3:37291 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-11-12T09:30:01,885 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:30:01,887 INFO [RS:0;jenkins-hbase3:37291 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37291 2023-11-12T09:30:01,899 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,37291,1699781316827] 2023-11-12T09:30:01,899 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,37291,1699781316827; numProcessing=1 2023-11-12T09:30:01,900 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase3.apache.org,37291,1699781316827 expired; onlineServers=0 2023-11-12T09:30:01,900 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,43745,1699781313745' ***** 2023-11-12T09:30:01,900 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-12T09:30:01,901 DEBUG [M:0;jenkins-hbase3:43745 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364ffcf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:30:01,901 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:30:01,958 INFO [M:0;jenkins-hbase3:43745 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@387df152{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:30:01,958 INFO [M:0;jenkins-hbase3:43745 {}] server.AbstractConnector(383): Stopped ServerConnector@3cc8b64c{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:30:01,958 INFO [M:0;jenkins-hbase3:43745 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:30:01,959 INFO [M:0;jenkins-hbase3:43745 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6709d0da{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:30:01,960 INFO [M:0;jenkins-hbase3:43745 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5c3f2af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/hadoop.log.dir/,STOPPED} 2023-11-12T09:30:01,960 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43745,1699781313745 2023-11-12T09:30:01,961 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43745,1699781313745; all regions closed. 2023-11-12T09:30:01,961 DEBUG [M:0;jenkins-hbase3:43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:01,961 INFO [M:0;jenkins-hbase3:43745 {}] master.HMaster(1666): Stopping master jetty server 2023-11-12T09:30:01,961 INFO [M:0;jenkins-hbase3:43745 {}] server.AbstractConnector(383): Stopped ServerConnector@481f5d5{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:30:01,962 DEBUG [M:0;jenkins-hbase3:43745 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-12T09:30:01,962 INFO [M:0;jenkins-hbase3:43745 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-12T09:30:01,962 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-12T09:30:01,962 DEBUG [M:0;jenkins-hbase3:43745 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:30:01,962 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781318605 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781318605,5,FailOnTimeoutGroup] 2023-11-12T09:30:01,962 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781318605 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781318605,5,FailOnTimeoutGroup] 2023-11-12T09:30:01,963 DEBUG [M:0;jenkins-hbase3:43745 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-12T09:30:01,963 INFO [M:0;jenkins-hbase3:43745 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-11-12T09:30:01,963 DEBUG [M:0;jenkins-hbase3:43745 {}] master.HMaster(1689): Stopping service threads 2023-11-12T09:30:01,963 INFO [M:0;jenkins-hbase3:43745 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-12T09:30:01,964 ERROR [M:0;jenkins-hbase3:43745 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-12T09:30:01,964 INFO [M:0;jenkins-hbase3:43745 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-12T09:30:01,964 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-12T09:30:01,965 WARN [M:0;jenkins-hbase3:43745 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-12T09:30:01,965 INFO [M:0;jenkins-hbase3:43745 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-12T09:30:01,965 INFO [M:0;jenkins-hbase3:43745 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-12T09:30:01,966 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:30:01,981 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,981 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,981 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:30:01,981 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:01,981 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=325.88 KB heapSize=389.62 KB 2023-11-12T09:30:01,994 DEBUG [M:0;jenkins-hbase3:43745 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c5651cf30d3427ba87ddda8e4c56fca is 88, key is hbase:meta,,1/info:sn/1699781319594/Put/seqid=0 2023-11-12T09:30:01,998 INFO [RS:0;jenkins-hbase3:37291 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37291,1699781316827; zookeeper connection closed. 2023-11-12T09:30:01,999 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c157c5a {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c157c5a 2023-11-12T09:30:01,999 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c5651cf30d3427ba87ddda8e4c56fca 2023-11-12T09:30:01,999 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-12T09:30:02,024 DEBUG [M:0;jenkins-hbase3:43745 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9992dc152d4d4c869f86473089f183cd is 1281, key is \x00\x00\x00\x00\x00\x00\x00\x19/proc:d/1699781336149/Put/seqid=0 2023-11-12T09:30:02,430 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=325.29 KB at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9992dc152d4d4c869f86473089f183cd 2023-11-12T09:30:02,437 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9992dc152d4d4c869f86473089f183cd 2023-11-12T09:30:02,453 DEBUG [M:0;jenkins-hbase3:43745 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24cfd257be534eb59049e6238e739c9e is 82, key is jenkins-hbase3.apache.org,37291,1699781316827/rs:state/1699781318711/Put/seqid=0 2023-11-12T09:30:02,860 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24cfd257be534eb59049e6238e739c9e 2023-11-12T09:30:02,866 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c5651cf30d3427ba87ddda8e4c56fca as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c5651cf30d3427ba87ddda8e4c56fca 2023-11-12T09:30:02,870 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c5651cf30d3427ba87ddda8e4c56fca, entries=8, sequenceid=758, filesize=5.5 K 2023-11-12T09:30:02,871 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9992dc152d4d4c869f86473089f183cd as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9992dc152d4d4c869f86473089f183cd 2023-11-12T09:30:02,882 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9992dc152d4d4c869f86473089f183cd 2023-11-12T09:30:02,882 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9992dc152d4d4c869f86473089f183cd, entries=83, sequenceid=758, filesize=26.0 K 2023-11-12T09:30:02,883 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24cfd257be534eb59049e6238e739c9e as hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24cfd257be534eb59049e6238e739c9e 2023-11-12T09:30:02,890 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36805/user/jenkins/test-data/fbe3aaad-091d-5e1e-2e94-8ff15f06b69c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24cfd257be534eb59049e6238e739c9e, entries=1, sequenceid=758, filesize=5.1 K 2023-11-12T09:30:02,891 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(3022): Finished flush of dataSize ~325.88 KB/333705, heapSize ~389.32 KB/398664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 910ms, sequenceid=758, compaction requested=false 2023-11-12T09:30:02,904 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:30:02,904 DEBUG [M:0;jenkins-hbase3:43745 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:30:02,919 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:30:02,919 INFO [M:0;jenkins-hbase3:43745 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-12T09:30:02,919 INFO [M:0;jenkins-hbase3:43745 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-12T09:30:02,920 INFO [M:0;jenkins-hbase3:43745 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:43745 2023-11-12T09:30:03,023 INFO [M:0;jenkins-hbase3:43745 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43745,1699781313745; zookeeper connection closed. 2023-11-12T09:30:03,029 WARN [Listener at jenkins-hbase3.apache.org/42507 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:30:03,033 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:30:03,142 WARN [BP-2054777967-172.31.12.81-1699781310530 heartbeating to localhost/127.0.0.1:36805 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:30:03,142 WARN [BP-2054777967-172.31.12.81-1699781310530 heartbeating to localhost/127.0.0.1:36805 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2054777967-172.31.12.81-1699781310530 (Datanode Uuid 4e120f22-d423-4d2b-aaab-dd185ce34744) service to localhost/127.0.0.1:36805 2023-11-12T09:30:03,144 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/dfs/data/data1/current/BP-2054777967-172.31.12.81-1699781310530 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:30:03,144 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b994a873-55b6-a31c-3ac2-86770e71eaf2/cluster_5f87ddc3-a8ea-ee58-a933-287057d95b74/dfs/data/data2/current/BP-2054777967-172.31.12.81-1699781310530 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:30:03,170 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:30:03,356 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-12T09:30:03,356 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-12T09:30:03,368 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:30:03,487 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:30:03,595 ERROR [Thread[Thread-277,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:30:03,595 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:30:03,698 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-12T09:30:03,702 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-12T09:30:03,703 ERROR [Thread[Thread-287,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:30:03,708 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:30:03,810 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:30:03,811 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:30:03,811 INFO [Listener at jenkins-hbase3.apache.org/42507 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped