2023-10-02 12:33:15,607 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa 2023-10-02 12:33:15,622 main DEBUG Took 0.012468 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:33:15,623 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:33:15,623 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:33:15,624 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:33:15,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,635 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:33:15,652 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,655 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,655 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,656 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,658 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,658 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,659 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,660 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,660 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,661 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,661 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,662 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,663 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,663 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,664 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,665 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,666 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:33:15,667 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,667 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:33:15,669 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:33:15,672 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:33:15,673 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:33:15,674 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:33:15,675 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:33:15,675 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:33:15,687 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:33:15,690 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:33:15,694 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:33:15,694 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:33:15,695 main DEBUG createAppenders(={Console}) 2023-10-02 12:33:15,695 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa initialized 2023-10-02 12:33:15,696 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa 2023-10-02 12:33:15,696 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a5baa OK. 2023-10-02 12:33:15,696 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:33:15,697 main DEBUG OutputStream closed 2023-10-02 12:33:15,697 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:33:15,697 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:33:15,697 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@72f926e6 OK 2023-10-02 12:33:15,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:33:15,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:33:15,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:33:15,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:33:15,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:33:15,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:33:15,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:33:15,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:33:15,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:33:15,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:33:15,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:33:15,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:33:15,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:33:15,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:33:15,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:33:15,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:33:15,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:33:15,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:33:15,753 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:33:15,753 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@2dde1bff) with optional ClassLoader: null 2023-10-02 12:33:15,753 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:33:15,754 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@2dde1bff] started OK. 2023-10-02T12:33:15,787 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupDelete timeout: 13 mins 2023-10-02 12:33:15,791 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:33:15,791 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:33:16,133 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845 2023-10-02T12:33:16,138 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:33:16,140 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:33:16,141 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:33:16,144 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830, deleteOnExit=true 2023-10-02T12:33:16,145 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:33:16,145 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/test.cache.data in system properties and HBase conf 2023-10-02T12:33:16,146 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:33:16,146 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:33:16,146 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:33:16,147 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:33:16,147 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:33:16,243 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:33:16,678 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:33:16,683 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:16,683 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:16,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:33:16,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:16,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:33:16,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:33:16,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:16,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:16,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:33:16,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:33:16,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:33:16,688 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:16,688 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:33:16,689 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:33:17,161 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:33:17,166 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:33:17,457 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:33:17,617 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@265fbe8e via org.mortbay.log.Slf4jLog 2023-10-02T12:33:17,639 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:17,699 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:17,738 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir/Jetty_localhost_41553_hdfs____59ivmo/webapp 2023-10-02T12:33:17,873 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41553 2023-10-02T12:33:17,901 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:33:17,902 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:33:18,466 WARN [Listener at localhost/36711 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:33:18,533 WARN [Listener at localhost/36711 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:33:18,553 WARN [Listener at localhost/36711 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:18,559 INFO [Listener at localhost/36711 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:18,564 INFO [Listener at localhost/36711 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir/Jetty_localhost_36505_datanode____.uj784i/webapp 2023-10-02T12:33:18,667 INFO [Listener at localhost/36711 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36505 2023-10-02T12:33:19,018 WARN [Listener at localhost/33177 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:33:19,331 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:33:19,544 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x2915cf67d0152347: Processing first storage report for DS-feec64fc-1707-4319-84a0-afbe06c78959 from datanode 1cc1d88c-4323-4d6a-95ba-5ffdfafaf159 2023-10-02T12:33:19,546 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x2915cf67d0152347: from storage DS-feec64fc-1707-4319-84a0-afbe06c78959 node DatanodeRegistration(127.0.0.1:44529, datanodeUuid=1cc1d88c-4323-4d6a-95ba-5ffdfafaf159, infoPort=37719, infoSecurePort=0, ipcPort=33177, storageInfo=lv=-57;cid=testClusterID;nsid=2102073901;c=1696249997240), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2023-10-02T12:33:19,547 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x2915cf67d0152347: Processing first storage report for DS-00c2fc02-e08b-491d-bdb3-9c8352713c7d from datanode 1cc1d88c-4323-4d6a-95ba-5ffdfafaf159 2023-10-02T12:33:19,548 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x2915cf67d0152347: from storage DS-00c2fc02-e08b-491d-bdb3-9c8352713c7d node DatanodeRegistration(127.0.0.1:44529, datanodeUuid=1cc1d88c-4323-4d6a-95ba-5ffdfafaf159, infoPort=37719, infoSecurePort=0, ipcPort=33177, storageInfo=lv=-57;cid=testClusterID;nsid=2102073901;c=1696249997240), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:33:19,566 DEBUG [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845 2023-10-02T12:33:19,666 INFO [Listener at localhost/33177 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/zookeeper_0, clientPort=65053, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:33:19,686 INFO [Listener at localhost/33177 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=65053 2023-10-02T12:33:19,694 INFO [Listener at localhost/33177 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:19,695 INFO [Listener at localhost/33177 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:20,381 INFO [Listener at localhost/33177 {}] util.FSUtils(462): Created version file at hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 with version=8 2023-10-02T12:33:20,381 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/hbase-staging 2023-10-02T12:33:20,398 DEBUG [Listener at localhost/33177 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:33:20,398 DEBUG [Listener at localhost/33177 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:33:20,398 DEBUG [Listener at localhost/33177 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:33:20,398 DEBUG [Listener at localhost/33177 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:33:20,637 DEBUG [Listener at localhost/33177 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:33:21,424 INFO [Listener at localhost/33177 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:33:21,473 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:33:21,474 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:33:21,474 INFO [Listener at localhost/33177 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:33:21,475 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:33:21,475 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:33:21,657 INFO [Listener at localhost/33177 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:33:21,796 INFO [Listener at localhost/33177 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:33:21,815 DEBUG [Listener at localhost/33177 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:33:21,822 INFO [Listener at localhost/33177 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:33:21,885 DEBUG [Listener at localhost/33177 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 30506 (auto-detected) 2023-10-02T12:33:21,888 DEBUG [Listener at localhost/33177 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:33:21,928 INFO [Listener at localhost/33177 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37819 2023-10-02T12:33:21,943 INFO [Listener at localhost/33177 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:21,945 INFO [Listener at localhost/33177 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:21,966 INFO [Listener at localhost/33177 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:37819 connecting to ZooKeeper ensemble=127.0.0.1:65053 2023-10-02T12:33:22,015 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:378190x0, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:33:22,019 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:37819-0x101732a517a0000 connected 2023-10-02T12:33:22,050 DEBUG [Listener at localhost/33177 {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:33:22,053 DEBUG [Listener at localhost/33177 {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:33:22,057 DEBUG [Listener at localhost/33177 {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:33:22,068 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37819 2023-10-02T12:33:22,069 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37819 2023-10-02T12:33:22,069 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37819 2023-10-02T12:33:22,070 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37819 2023-10-02T12:33:22,070 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37819 2023-10-02T12:33:22,125 INFO [Listener at localhost/33177 {}] log.Log(170): Logging initialized @7256ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:33:22,249 INFO [Listener at localhost/33177 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:33:22,250 INFO [Listener at localhost/33177 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:33:22,251 INFO [Listener at localhost/33177 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:33:22,253 INFO [Listener at localhost/33177 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:33:22,253 INFO [Listener at localhost/33177 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:33:22,253 INFO [Listener at localhost/33177 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:33:22,256 INFO [Listener at localhost/33177 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:33:22,404 INFO [Listener at localhost/33177 {}] http.HttpServer(1219): Jetty bound to port 44567 2023-10-02T12:33:22,407 INFO [Listener at localhost/33177 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:33:22,451 INFO [Listener at localhost/33177 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:33:22,456 INFO [Listener at localhost/33177 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4ebc82c4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:33:22,457 INFO [Listener at localhost/33177 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:33:22,457 INFO [Listener at localhost/33177 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5fe61d87{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:33:22,534 INFO [Listener at localhost/33177 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:33:22,547 INFO [Listener at localhost/33177 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:33:22,547 INFO [Listener at localhost/33177 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:33:22,549 INFO [Listener at localhost/33177 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:33:22,557 INFO [Listener at localhost/33177 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:33:23,752 INFO [Listener at localhost/33177 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@210ce6e1{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:33:23,765 INFO [Listener at localhost/33177 {}] server.AbstractConnector(333): Started ServerConnector@761be04a{HTTP/1.1, (http/1.1)}{0.0.0.0:44567} 2023-10-02T12:33:23,766 INFO [Listener at localhost/33177 {}] server.Server(415): Started @8896ms 2023-10-02T12:33:23,770 INFO [Listener at localhost/33177 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664, hbase.cluster.distributed=false 2023-10-02T12:33:23,833 INFO [Listener at localhost/33177 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:33:23,834 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:33:23,834 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:33:23,834 INFO [Listener at localhost/33177 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:33:23,834 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:33:23,834 INFO [Listener at localhost/33177 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:33:23,843 INFO [Listener at localhost/33177 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:33:23,845 INFO [Listener at localhost/33177 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:33:23,846 INFO [Listener at localhost/33177 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:40937 2023-10-02T12:33:23,849 INFO [Listener at localhost/33177 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:33:23,857 DEBUG [Listener at localhost/33177 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:33:23,858 INFO [Listener at localhost/33177 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:23,861 INFO [Listener at localhost/33177 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:23,863 INFO [Listener at localhost/33177 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:40937 connecting to ZooKeeper ensemble=127.0.0.1:65053 2023-10-02T12:33:23,867 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:409370x0, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:33:23,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:40937-0x101732a517a0001 connected 2023-10-02T12:33:23,868 DEBUG [Listener at localhost/33177 {}] zookeeper.ZKUtil(113): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:33:23,870 DEBUG [Listener at localhost/33177 {}] zookeeper.ZKUtil(113): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:33:23,871 DEBUG [Listener at localhost/33177 {}] zookeeper.ZKUtil(113): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:33:23,872 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40937 2023-10-02T12:33:23,872 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40937 2023-10-02T12:33:23,874 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40937 2023-10-02T12:33:23,878 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40937 2023-10-02T12:33:23,878 DEBUG [Listener at localhost/33177 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40937 2023-10-02T12:33:23,885 INFO [Listener at localhost/33177 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:33:23,886 INFO [Listener at localhost/33177 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:33:23,886 INFO [Listener at localhost/33177 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:33:23,887 INFO [Listener at localhost/33177 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:33:23,887 INFO [Listener at localhost/33177 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:33:23,888 INFO [Listener at localhost/33177 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:33:23,888 INFO [Listener at localhost/33177 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:33:23,891 INFO [Listener at localhost/33177 {}] http.HttpServer(1219): Jetty bound to port 37643 2023-10-02T12:33:23,891 INFO [Listener at localhost/33177 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:33:23,898 INFO [Listener at localhost/33177 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:33:23,899 INFO [Listener at localhost/33177 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@21a8923c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:33:23,899 INFO [Listener at localhost/33177 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:33:23,899 INFO [Listener at localhost/33177 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1b05b10e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:33:23,917 INFO [Listener at localhost/33177 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:33:23,918 INFO [Listener at localhost/33177 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:33:23,919 INFO [Listener at localhost/33177 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:33:23,919 INFO [Listener at localhost/33177 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:33:23,921 INFO [Listener at localhost/33177 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:33:23,926 INFO [Listener at localhost/33177 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@610d16b9{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:33:23,927 INFO [Listener at localhost/33177 {}] server.AbstractConnector(333): Started ServerConnector@36efd4d1{HTTP/1.1, (http/1.1)}{0.0.0.0:37643} 2023-10-02T12:33:23,927 INFO [Listener at localhost/33177 {}] server.Server(415): Started @9058ms 2023-10-02T12:33:23,933 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:33:23,948 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@5a1ea680{HTTP/1.1, (http/1.1)}{0.0.0.0:41999} 2023-10-02T12:33:23,948 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9079ms 2023-10-02T12:33:23,948 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:23,957 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:37819 2023-10-02T12:33:23,961 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:33:23,961 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:33:23,963 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:23,983 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:33:23,983 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:33:23,983 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:23,984 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:23,985 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:33:23,986 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,37819,1696250000621 from backup master directory 2023-10-02T12:33:23,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:33:23,992 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:33:23,992 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:23,992 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:33:23,993 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:33:23,993 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:23,996 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:33:23,998 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:33:24,090 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/hbase.id with ID: 1322a032-5fcd-47d2-af0a-2cf3f66b7f72 2023-10-02T12:33:24,140 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:33:24,159 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:24,159 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:24,218 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:24,222 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:33:24,279 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:33:24,280 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:33:24,281 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:33:24,295 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:33:24,296 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:33:24,397 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store 2023-10-02T12:33:24,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:33:24,447 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:24,448 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:33:24,448 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:24,448 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:24,448 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:33:24,448 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:24,448 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:33:24,449 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:33:24,451 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/.initializing 2023-10-02T12:33:24,451 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/WALs/jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:24,463 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:33:24,479 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37819%2C1696250000621, suffix=, logDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/WALs/jenkins-hbase3.apache.org,37819,1696250000621, archiveDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/oldWALs, maxLogs=10 2023-10-02T12:33:24,518 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/WALs/jenkins-hbase3.apache.org,37819,1696250000621/jenkins-hbase3.apache.org%2C37819%2C1696250000621.1696250004486, exclude list is [], retry=0 2023-10-02T12:33:24,553 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:24,562 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:33:24,683 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/WALs/jenkins-hbase3.apache.org,37819,1696250000621/jenkins-hbase3.apache.org%2C37819%2C1696250000621.1696250004486 2023-10-02T12:33:24,683 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:24,684 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:24,685 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:24,691 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,693 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:33:24,853 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:24,858 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:24,858 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:33:24,863 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:24,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:24,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:33:24,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:24,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:24,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:33:24,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:24,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:24,887 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,889 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,909 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:33:24,913 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:33:24,918 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:24,919 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11062373760, jitterRate=0.030263841152191162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:33:24,929 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:33:24,930 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:33:24,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600f3ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:25,031 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:33:25,053 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:33:25,053 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:33:25,057 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:33:25,059 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-02T12:33:25,067 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 7 msec 2023-10-02T12:33:25,067 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:33:25,106 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:33:25,123 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:33:25,126 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:33:25,129 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:33:25,131 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:33:25,133 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:33:25,136 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:33:25,143 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:33:25,144 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:33:25,146 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:33:25,148 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:33:25,166 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:33:25,168 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:33:25,174 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:33:25,174 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,174 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:33:25,174 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,174 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,37819,1696250000621, sessionid=0x101732a517a0000, setting cluster-up flag (Was=false) 2023-10-02T12:33:25,189 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:33:25,199 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,199 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,204 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:33:25,205 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:25,210 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,210 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,216 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:33:25,217 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:25,222 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,222 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:25,228 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:33:25,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:25,281 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:40937 2023-10-02T12:33:25,282 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(997): ClusterId : 1322a032-5fcd-47d2-af0a-2cf3f66b7f72 2023-10-02T12:33:25,285 INFO [RS:0;jenkins-hbase3:40937 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:33:25,289 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:33:25,296 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:33:25,296 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:33:25,300 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:33:25,300 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:33:25,304 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:33:25,305 DEBUG [RS:0;jenkins-hbase3:40937 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d822d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:25,314 DEBUG [RS:0;jenkins-hbase3:40937 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547ae173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:33:25,321 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:33:25,322 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:33:25,322 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:33:25,326 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,37819,1696250000621 with isa=jenkins-hbase3.apache.org/172.31.12.81:40937, startcode=1696250003832 2023-10-02T12:33:25,353 DEBUG [RS:0;jenkins-hbase3:40937 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:33:25,422 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:33:25,436 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:33:25,436 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48011, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:33:25,445 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:33:25,448 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37819 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:33:25,457 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,37819,1696250000621 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:33:25,464 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:25,464 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:25,465 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:25,465 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:33:25,465 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:33:25,465 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,465 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:33:25,466 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,479 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696250035479 2023-10-02T12:33:25,484 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:33:25,487 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:33:25,488 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:33:25,489 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:33:25,495 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:25,495 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:33:25,500 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-10-02T12:33:25,500 WARN [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-10-02T12:33:25,504 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:33:25,505 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:33:25,506 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:33:25,507 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:33:25,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:33:25,518 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,522 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:33:25,525 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:33:25,528 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:33:25,539 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47483913 to 127.0.0.1:65053 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:25,548 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66edb222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:25,548 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:33:25,552 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:33:25,552 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:33:25,554 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250005554,5,FailOnTimeoutGroup] 2023-10-02T12:33:25,554 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250005554,5,FailOnTimeoutGroup] 2023-10-02T12:33:25,554 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:33:25,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,601 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,37819,1696250000621 with isa=jenkins-hbase3.apache.org/172.31.12.81:40937, startcode=1696250003832 2023-10-02T12:33:25,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37819 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:25,622 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:25,622 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:36711 2023-10-02T12:33:25,623 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44567 2023-10-02T12:33:25,629 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:33:25,630 DEBUG [RS:0;jenkins-hbase3:40937 {}] zookeeper.ZKUtil(111): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:25,630 WARN [RS:0;jenkins-hbase3:40937 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:33:25,630 INFO [RS:0;jenkins-hbase3:40937 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:33:25,630 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:25,634 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,40937,1696250003832] 2023-10-02T12:33:25,652 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:33:25,673 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:33:25,693 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:33:25,699 INFO [RS:0;jenkins-hbase3:40937 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:33:25,700 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,701 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:33:25,714 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,715 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,715 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,715 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,715 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,716 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,716 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:33:25,716 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,716 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,717 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,717 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,717 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:33:25,717 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:33:25,718 DEBUG [RS:0;jenkins-hbase3:40937 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:33:25,722 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,722 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,722 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,722 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,723 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40937,1696250003832-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:33:25,746 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:33:25,750 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40937,1696250003832-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:25,773 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,40937,1696250003832 started 2023-10-02T12:33:25,774 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,40937,1696250003832, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:40937, sessionid=0x101732a517a0001 2023-10-02T12:33:25,774 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:33:25,775 DEBUG [RS:0;jenkins-hbase3:40937 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:25,775 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40937,1696250003832' 2023-10-02T12:33:25,775 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:33:25,776 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:33:25,776 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:33:25,777 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:33:25,777 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40937,1696250003832' 2023-10-02T12:33:25,777 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:25,778 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:25,778 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:33:25,778 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:33:25,778 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:33:25,779 DEBUG [RS:0;jenkins-hbase3:40937 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:25,779 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40937,1696250003832' 2023-10-02T12:33:25,779 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:33:25,780 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:33:25,780 DEBUG [RS:0;jenkins-hbase3:40937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:33:25,781 INFO [RS:0;jenkins-hbase3:40937 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:33:25,781 INFO [RS:0;jenkins-hbase3:40937 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:33:25,891 INFO [RS:0;jenkins-hbase3:40937 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:33:25,894 INFO [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40937%2C1696250003832, suffix=, logDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832, archiveDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs, maxLogs=32 2023-10-02T12:33:25,906 DEBUG [RS:0;jenkins-hbase3:40937 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250005895, exclude list is [], retry=0 2023-10-02T12:33:25,912 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:25,916 INFO [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250005895 2023-10-02T12:33:25,917 DEBUG [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:25,930 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:33:25,930 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:26,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:26,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:33:26,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:33:26,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:26,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:26,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:33:26,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:33:26,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:26,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:26,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:33:26,363 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:33:26,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:26,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:26,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740 2023-10-02T12:33:26,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740 2023-10-02T12:33:26,371 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:33:26,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:33:26,379 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:26,380 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10231245600, jitterRate=-0.047141000628471375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:33:26,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:33:26,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:33:26,383 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:33:26,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:33:26,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:33:26,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:33:26,385 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:33:26,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:33:26,389 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:33:26,390 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:33:26,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:33:26,420 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:33:26,424 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:33:26,576 DEBUG [jenkins-hbase3:37819 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:33:26,586 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:26,592 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,40937,1696250003832, state=OPENING 2023-10-02T12:33:26,601 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:33:26,607 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:26,607 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:26,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:33:26,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:33:26,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:26,809 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:26,813 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:33:26,817 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57688, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:33:26,834 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:33:26,835 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:33:26,835 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:33:26,839 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta, suffix=.meta, logDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832, archiveDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs, maxLogs=32 2023-10-02T12:33:26,859 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250006841.meta, exclude list is [], retry=0 2023-10-02T12:33:26,864 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:26,869 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250006841.meta 2023-10-02T12:33:26,870 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:26,870 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:26,877 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:26,878 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:33:26,903 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:33:26,903 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:33:26,910 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:33:26,910 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:26,910 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:33:26,911 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:33:26,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:33:26,916 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:33:26,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:26,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:26,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:33:26,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:33:26,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:26,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:26,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:33:26,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:33:26,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:26,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:33:26,924 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740 2023-10-02T12:33:26,927 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740 2023-10-02T12:33:26,929 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:33:26,932 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:33:26,934 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10486857280, jitterRate=-0.023335307836532593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:33:26,936 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:33:26,953 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696250006798 2023-10-02T12:33:26,980 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:33:26,980 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:26,981 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:33:26,983 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,40937,1696250003832, state=OPEN 2023-10-02T12:33:26,986 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:33:26,986 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:33:26,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:33:26,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:33:27,134 INFO [AsyncFSWAL-0-hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 147 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:27,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:33:27,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,40937,1696250003832 in 375 msec 2023-10-02T12:33:27,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:33:27,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 740 msec 2023-10-02T12:33:27,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.8420 sec 2023-10-02T12:33:27,167 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696250007166, completionTime=-1 2023-10-02T12:33:27,167 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:33:27,167 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:33:27,243 DEBUG [hconnection-0x595731fb-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:27,247 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57704, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:27,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:33:27,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696250067266 2023-10-02T12:33:27,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696250127266 2023-10-02T12:33:27,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 99 msec 2023-10-02T12:33:27,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37819,1696250000621-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:27,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37819,1696250000621-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:27,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37819,1696250000621-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:27,294 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:37819, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:27,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:33:27,302 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:33:27,322 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:33:27,325 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:33:27,338 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:33:27,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:27,345 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:27,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:27,770 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => edb5468052a27c274e91621fff6dca47, NAME => 'hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:27,782 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:27,782 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing edb5468052a27c274e91621fff6dca47, disabling compactions & flushes 2023-10-02T12:33:27,783 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:27,783 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:27,783 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. after waiting 0 ms 2023-10-02T12:33:27,783 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:27,783 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:27,783 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for edb5468052a27c274e91621fff6dca47: 2023-10-02T12:33:27,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:27,799 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696250007787"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250007787"}]},"ts":"1696250007787"} 2023-10-02T12:33:27,843 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:27,846 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:27,854 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250007846"}]},"ts":"1696250007846"} 2023-10-02T12:33:27,859 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:33:27,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=edb5468052a27c274e91621fff6dca47, ASSIGN}] 2023-10-02T12:33:27,873 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=edb5468052a27c274e91621fff6dca47, ASSIGN 2023-10-02T12:33:27,878 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=edb5468052a27c274e91621fff6dca47, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:28,029 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=edb5468052a27c274e91621fff6dca47, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:28,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure edb5468052a27c274e91621fff6dca47, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:28,196 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:28,197 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => edb5468052a27c274e91621fff6dca47, NAME => 'hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:28,198 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:28,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:28,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,202 INFO [StoreOpener-edb5468052a27c274e91621fff6dca47-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,205 INFO [StoreOpener-edb5468052a27c274e91621fff6dca47-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edb5468052a27c274e91621fff6dca47 columnFamilyName info 2023-10-02T12:33:28,205 DEBUG [StoreOpener-edb5468052a27c274e91621fff6dca47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:28,206 INFO [StoreOpener-edb5468052a27c274e91621fff6dca47-1 {}] regionserver.HStore(324): Store=edb5468052a27c274e91621fff6dca47/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:28,210 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,211 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,219 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for edb5468052a27c274e91621fff6dca47 2023-10-02T12:33:28,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:28,225 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened edb5468052a27c274e91621fff6dca47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10496649440, jitterRate=-0.022423341870307922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:28,226 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for edb5468052a27c274e91621fff6dca47: 2023-10-02T12:33:28,229 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47., pid=6, masterSystemTime=1696250008189 2023-10-02T12:33:28,233 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:28,233 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:33:28,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=edb5468052a27c274e91621fff6dca47, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:28,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:33:28,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure edb5468052a27c274e91621fff6dca47, server=jenkins-hbase3.apache.org,40937,1696250003832 in 205 msec 2023-10-02T12:33:28,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:33:28,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=edb5468052a27c274e91621fff6dca47, ASSIGN in 374 msec 2023-10-02T12:33:28,249 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:28,249 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250008249"}]},"ts":"1696250008249"} 2023-10-02T12:33:28,253 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:33:28,256 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:28,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 930 msec 2023-10-02T12:33:28,343 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:33:28,344 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:28,344 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:28,345 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:33:28,406 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:33:28,443 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:28,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 54 msec 2023-10-02T12:33:28,460 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:33:28,477 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:28,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 22 msec 2023-10-02T12:33:28,500 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:33:28,503 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:33:28,515 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.522sec 2023-10-02T12:33:28,522 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:33:28,524 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:33:28,526 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:33:28,527 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:33:28,527 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:33:28,529 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37819,1696250000621-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:33:28,530 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37819,1696250000621-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:33:28,543 DEBUG [Listener at localhost/33177 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x050387ad to 127.0.0.1:65053 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:28,583 DEBUG [Listener at localhost/33177 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fef9536, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:28,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:33:28,589 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:33:28,589 DEBUG [Listener at localhost/33177 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:33:28,589 DEBUG [Listener at localhost/33177 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:33:28,610 DEBUG [hconnection-0x6eb07023-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:28,626 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57716, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:28,643 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:33:28,643 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/test.cache.data in system properties and HBase conf 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:33:28,644 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:33:28,645 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:33:28,646 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:33:28,646 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:33:28,646 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:33:28,646 INFO [Listener at localhost/33177 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:33:30,678 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:31,129 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:31,137 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36779_jobhistory____.viobhc/webapp 2023-10-02T12:33:31,337 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:33:31,338 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:31,338 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:31,342 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:31,438 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:31,582 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:31,662 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:33:31,664 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:33:31,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:33:31,794 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:31,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:33:31,804 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:31,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:33:31,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:33:31,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:33:31,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:33:31,809 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:33:31,809 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:31,964 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:33:32,013 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36779 2023-10-02T12:33:32,166 WARN [Listener at jenkins-hbase3.apache.org/33645 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:32,289 WARN [Listener at jenkins-hbase3.apache.org/33645 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:32,290 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:32,313 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_35121_cluster____1itvx1/webapp 2023-10-02T12:33:32,444 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:32,444 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:33:32,445 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:32,445 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:32,456 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:32,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:32,691 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:32,828 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:32,837 INFO [Listener at jenkins-hbase3.apache.org/33645 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35121 2023-10-02T12:33:33,395 WARN [Listener at jenkins-hbase3.apache.org/37967 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:33,429 WARN [Listener at jenkins-hbase3.apache.org/37967 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:33,430 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:33,438 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40067_node____uoke1g/webapp 2023-10-02T12:33:33,516 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:33:33,516 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:33,517 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:33,517 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:33,529 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:33,556 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:33,636 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:33,647 INFO [Listener at jenkins-hbase3.apache.org/37967 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40067 2023-10-02T12:33:33,816 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:33:33,825 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:33:33,826 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:33:33,832 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34571_node____jxdyl/webapp 2023-10-02T12:33:33,885 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:33:33,885 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:33:33,885 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:33:33,885 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:33:33,893 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:33,912 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:33,979 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:33:33,986 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34571 2023-10-02T12:33:34,034 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:33:34,034 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:36711/backupUT 2023-10-02T12:33:34,051 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:34,055 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38732, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:34,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:33:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:33:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:33:34,095 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:34,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 29 msec 2023-10-02T12:33:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:33:34,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:33:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:33:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:33:34,208 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:34,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 20 msec 2023-10-02T12:33:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:33:34,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:33:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:33:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:33:34,319 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:34,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 19 msec 2023-10-02T12:33:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:33:34,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:33:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:33:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:33:34,432 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:34,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 20 msec 2023-10-02T12:33:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:33:34,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696250014034', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696250014034 2023-10-02T12:33:34,546 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696250014034 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:34,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696250014034" procId is: 13 2023-10-02T12:33:34,547 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:34,548 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696250014034 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:34,987 INFO [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0d134e56945ef6d5e83667f94878fc5e, NAME => 'test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696250014034', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:35,407 DEBUG [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:35,407 DEBUG [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1668): Closing 0d134e56945ef6d5e83667f94878fc5e, disabling compactions & flushes 2023-10-02T12:33:35,407 INFO [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,407 DEBUG [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,407 DEBUG [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. after waiting 0 ms 2023-10-02T12:33:35,407 DEBUG [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,407 INFO [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1909): Closed test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,407 DEBUG [RegionOpenAndInit-test-1696250014034-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0d134e56945ef6d5e83667f94878fc5e: 2023-10-02T12:33:35,409 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696250014034 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:35,409 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696250015409"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250015409"}]},"ts":"1696250015409"} 2023-10-02T12:33:35,412 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:35,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696250014034 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:35,414 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696250014034","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250015414"}]},"ts":"1696250015414"} 2023-10-02T12:33:35,416 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696250014034, state=ENABLING in hbase:meta 2023-10-02T12:33:35,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696250014034, region=0d134e56945ef6d5e83667f94878fc5e, ASSIGN}] 2023-10-02T12:33:35,423 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696250014034, region=0d134e56945ef6d5e83667f94878fc5e, ASSIGN 2023-10-02T12:33:35,424 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696250014034, region=0d134e56945ef6d5e83667f94878fc5e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:35,576 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=0d134e56945ef6d5e83667f94878fc5e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:35,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 0d134e56945ef6d5e83667f94878fc5e, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:35,743 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 0d134e56945ef6d5e83667f94878fc5e, NAME => 'test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:35,743 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:35,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696250014034 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:35,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,747 INFO [StoreOpener-0d134e56945ef6d5e83667f94878fc5e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,750 INFO [StoreOpener-0d134e56945ef6d5e83667f94878fc5e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d134e56945ef6d5e83667f94878fc5e columnFamilyName f 2023-10-02T12:33:35,750 DEBUG [StoreOpener-0d134e56945ef6d5e83667f94878fc5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:35,751 INFO [StoreOpener-0d134e56945ef6d5e83667f94878fc5e-1 {}] regionserver.HStore(324): Store=0d134e56945ef6d5e83667f94878fc5e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:35,752 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,753 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:35,763 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:35,764 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 0d134e56945ef6d5e83667f94878fc5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10080355680, jitterRate=-0.06119371950626373}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:35,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 0d134e56945ef6d5e83667f94878fc5e: 2023-10-02T12:33:35,766 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e., pid=15, masterSystemTime=1696250015733 2023-10-02T12:33:35,769 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,769 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:35,770 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=0d134e56945ef6d5e83667f94878fc5e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:35,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:33:35,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 0d134e56945ef6d5e83667f94878fc5e, server=jenkins-hbase3.apache.org,40937,1696250003832 in 194 msec 2023-10-02T12:33:35,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:33:35,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696250014034, region=0d134e56945ef6d5e83667f94878fc5e, ASSIGN in 355 msec 2023-10-02T12:33:35,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696250014034 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:35,781 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696250014034","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250015780"}]},"ts":"1696250015780"} 2023-10-02T12:33:35,783 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696250014034, state=ENABLED in hbase:meta 2023-10-02T12:33:35,786 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696250014034 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:35,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696250014034 in 1.2440 sec 2023-10-02T12:33:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:33:36,668 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696250014034, procId: 13 completed 2023-10-02T12:33:36,670 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5c379750 to 127.0.0.1:65053 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:36,678 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@487ce5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:36,680 DEBUG [hconnection-0xb89e1b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:36,685 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:36,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40937 {}] regionserver.HRegion(8210): writing data to region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:33:36,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962500140341', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962500140341 2023-10-02T12:33:36,816 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962500140341 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:36,816 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:36,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962500140341" procId is: 16 2023-10-02T12:33:36,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962500140341 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:37,236 INFO [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 639d2bc3c5c2cb1c241de32b99c9da77, NAME => 'ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962500140341', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:37,255 DEBUG [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:37,255 DEBUG [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1668): Closing 639d2bc3c5c2cb1c241de32b99c9da77, disabling compactions & flushes 2023-10-02T12:33:37,256 INFO [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,256 DEBUG [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,256 DEBUG [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. after waiting 0 ms 2023-10-02T12:33:37,256 DEBUG [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,256 INFO [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,256 DEBUG [RegionOpenAndInit-ns2:test-16962500140341-pool-0 {}] regionserver.HRegion(1622): Region close journal for 639d2bc3c5c2cb1c241de32b99c9da77: 2023-10-02T12:33:37,265 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962500140341 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:37,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250017265"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250017265"}]},"ts":"1696250017265"} 2023-10-02T12:33:37,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:37,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962500140341 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:37,270 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962500140341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250017270"}]},"ts":"1696250017270"} 2023-10-02T12:33:37,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962500140341, state=ENABLING in hbase:meta 2023-10-02T12:33:37,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962500140341, region=639d2bc3c5c2cb1c241de32b99c9da77, ASSIGN}] 2023-10-02T12:33:37,280 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962500140341, region=639d2bc3c5c2cb1c241de32b99c9da77, ASSIGN 2023-10-02T12:33:37,281 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962500140341, region=639d2bc3c5c2cb1c241de32b99c9da77, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:37,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=639d2bc3c5c2cb1c241de32b99c9da77, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:37,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 639d2bc3c5c2cb1c241de32b99c9da77, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:37,593 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,593 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 639d2bc3c5c2cb1c241de32b99c9da77, NAME => 'ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:37,593 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:37,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962500140341 639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:37,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,596 INFO [StoreOpener-639d2bc3c5c2cb1c241de32b99c9da77-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,598 INFO [StoreOpener-639d2bc3c5c2cb1c241de32b99c9da77-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 639d2bc3c5c2cb1c241de32b99c9da77 columnFamilyName f 2023-10-02T12:33:37,598 DEBUG [StoreOpener-639d2bc3c5c2cb1c241de32b99c9da77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:37,599 INFO [StoreOpener-639d2bc3c5c2cb1c241de32b99c9da77-1 {}] regionserver.HStore(324): Store=639d2bc3c5c2cb1c241de32b99c9da77/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:37,600 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,601 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:33:37,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:37,612 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 639d2bc3c5c2cb1c241de32b99c9da77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10715654240, jitterRate=-0.0020269304513931274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:37,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 639d2bc3c5c2cb1c241de32b99c9da77: 2023-10-02T12:33:37,614 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77., pid=18, masterSystemTime=1696250017588 2023-10-02T12:33:37,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,617 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:33:37,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=639d2bc3c5c2cb1c241de32b99c9da77, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:37,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:33:37,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 639d2bc3c5c2cb1c241de32b99c9da77, server=jenkins-hbase3.apache.org,40937,1696250003832 in 184 msec 2023-10-02T12:33:37,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:33:37,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962500140341, region=639d2bc3c5c2cb1c241de32b99c9da77, ASSIGN in 347 msec 2023-10-02T12:33:37,627 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962500140341 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:37,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962500140341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250017627"}]},"ts":"1696250017627"} 2023-10-02T12:33:37,630 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962500140341, state=ENABLED in hbase:meta 2023-10-02T12:33:37,633 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962500140341 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:37,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962500140341 in 821 msec 2023-10-02T12:33:37,665 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696250014034' 2023-10-02T12:33:37,666 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962500140341' 2023-10-02T12:33:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:33:37,926 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962500140341, procId: 16 completed 2023-10-02T12:33:37,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40937 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:33:38,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962500140342', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962500140342 2023-10-02T12:33:38,021 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962500140342 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:38,021 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:38,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962500140342" procId is: 19 2023-10-02T12:33:38,023 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962500140342 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:38,457 INFO [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 252a897416193ae7ac42e7e4f2a3268e, NAME => 'ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962500140342', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:38,874 DEBUG [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:38,874 DEBUG [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1668): Closing 252a897416193ae7ac42e7e4f2a3268e, disabling compactions & flushes 2023-10-02T12:33:38,874 INFO [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:38,874 DEBUG [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:38,875 DEBUG [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. after waiting 0 ms 2023-10-02T12:33:38,875 DEBUG [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:38,875 INFO [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:38,875 DEBUG [RegionOpenAndInit-ns3:test-16962500140342-pool-0 {}] regionserver.HRegion(1622): Region close journal for 252a897416193ae7ac42e7e4f2a3268e: 2023-10-02T12:33:38,876 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962500140342 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:38,877 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250018876"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250018876"}]},"ts":"1696250018876"} 2023-10-02T12:33:38,879 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:38,881 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962500140342 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:38,881 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962500140342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250018881"}]},"ts":"1696250018881"} 2023-10-02T12:33:38,883 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962500140342, state=ENABLING in hbase:meta 2023-10-02T12:33:38,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962500140342, region=252a897416193ae7ac42e7e4f2a3268e, ASSIGN}] 2023-10-02T12:33:38,891 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962500140342, region=252a897416193ae7ac42e7e4f2a3268e, ASSIGN 2023-10-02T12:33:38,892 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962500140342, region=252a897416193ae7ac42e7e4f2a3268e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:39,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=252a897416193ae7ac42e7e4f2a3268e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:39,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 252a897416193ae7ac42e7e4f2a3268e, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:39,205 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:39,205 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 252a897416193ae7ac42e7e4f2a3268e, NAME => 'ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:39,205 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:39,206 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962500140342 252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,206 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:39,206 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,206 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,208 INFO [StoreOpener-252a897416193ae7ac42e7e4f2a3268e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,210 INFO [StoreOpener-252a897416193ae7ac42e7e4f2a3268e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 252a897416193ae7ac42e7e4f2a3268e columnFamilyName f 2023-10-02T12:33:39,210 DEBUG [StoreOpener-252a897416193ae7ac42e7e4f2a3268e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:39,211 INFO [StoreOpener-252a897416193ae7ac42e7e4f2a3268e-1 {}] regionserver.HStore(324): Store=252a897416193ae7ac42e7e4f2a3268e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:39,212 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns3/test-16962500140342/252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,213 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns3/test-16962500140342/252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,216 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:33:39,219 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns3/test-16962500140342/252a897416193ae7ac42e7e4f2a3268e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:39,220 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 252a897416193ae7ac42e7e4f2a3268e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9553223040, jitterRate=-0.11028677225112915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:39,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 252a897416193ae7ac42e7e4f2a3268e: 2023-10-02T12:33:39,222 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e., pid=21, masterSystemTime=1696250019199 2023-10-02T12:33:39,225 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:39,225 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:33:39,226 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=252a897416193ae7ac42e7e4f2a3268e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:39,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:33:39,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 252a897416193ae7ac42e7e4f2a3268e, server=jenkins-hbase3.apache.org,40937,1696250003832 in 182 msec 2023-10-02T12:33:39,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:33:39,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962500140342, region=252a897416193ae7ac42e7e4f2a3268e, ASSIGN in 343 msec 2023-10-02T12:33:39,235 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962500140342 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:39,236 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962500140342","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250019235"}]},"ts":"1696250019235"} 2023-10-02T12:33:39,237 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962500140342, state=ENABLED in hbase:meta 2023-10-02T12:33:39,240 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962500140342 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:39,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962500140342 in 1.2230 sec 2023-10-02T12:33:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:33:40,133 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962500140342, procId: 19 completed 2023-10-02T12:33:40,134 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962500140342 get assigned. Timeout = 60000ms 2023-10-02T12:33:40,136 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:40,141 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962500140342 assigned to meta. Checking AM states. 2023-10-02T12:33:40,142 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:40,142 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962500140342 assigned. 2023-10-02T12:33:40,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962500140343', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962500140343 2023-10-02T12:33:40,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962500140343 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:40,149 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:40,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962500140343" procId is: 22 2023-10-02T12:33:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:40,156 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962500140343 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:40,598 INFO [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3b06f8973540cb8d6705c60d9447cf6c, NAME => 'ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962500140343', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:40,623 DEBUG [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:40,624 DEBUG [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1668): Closing 3b06f8973540cb8d6705c60d9447cf6c, disabling compactions & flushes 2023-10-02T12:33:40,624 INFO [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,624 DEBUG [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,624 DEBUG [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. after waiting 0 ms 2023-10-02T12:33:40,624 DEBUG [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,624 INFO [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,624 DEBUG [RegionOpenAndInit-ns4:test-16962500140343-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3b06f8973540cb8d6705c60d9447cf6c: 2023-10-02T12:33:40,625 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962500140343 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:40,626 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250020626"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250020626"}]},"ts":"1696250020626"} 2023-10-02T12:33:40,629 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:40,630 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962500140343 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:40,630 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962500140343","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250020630"}]},"ts":"1696250020630"} 2023-10-02T12:33:40,632 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962500140343, state=ENABLING in hbase:meta 2023-10-02T12:33:40,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962500140343, region=3b06f8973540cb8d6705c60d9447cf6c, ASSIGN}] 2023-10-02T12:33:40,640 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962500140343, region=3b06f8973540cb8d6705c60d9447cf6c, ASSIGN 2023-10-02T12:33:40,641 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962500140343, region=3b06f8973540cb8d6705c60d9447cf6c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:40,793 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=3b06f8973540cb8d6705c60d9447cf6c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:40,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 3b06f8973540cb8d6705c60d9447cf6c, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:40,956 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,956 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 3b06f8973540cb8d6705c60d9447cf6c, NAME => 'ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:40,957 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:40,957 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962500140343 3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,957 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:40,957 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,957 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,959 INFO [StoreOpener-3b06f8973540cb8d6705c60d9447cf6c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,962 INFO [StoreOpener-3b06f8973540cb8d6705c60d9447cf6c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b06f8973540cb8d6705c60d9447cf6c columnFamilyName f 2023-10-02T12:33:40,962 DEBUG [StoreOpener-3b06f8973540cb8d6705c60d9447cf6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:40,963 INFO [StoreOpener-3b06f8973540cb8d6705c60d9447cf6c-1 {}] regionserver.HStore(324): Store=3b06f8973540cb8d6705c60d9447cf6c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:40,964 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns4/test-16962500140343/3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,965 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns4/test-16962500140343/3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,968 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:33:40,972 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns4/test-16962500140343/3b06f8973540cb8d6705c60d9447cf6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:40,973 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 3b06f8973540cb8d6705c60d9447cf6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11182969920, jitterRate=0.04149523377418518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:33:40,973 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 3b06f8973540cb8d6705c60d9447cf6c: 2023-10-02T12:33:40,974 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c., pid=24, masterSystemTime=1696250020950 2023-10-02T12:33:40,977 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,977 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:33:40,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=3b06f8973540cb8d6705c60d9447cf6c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:40,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:33:40,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 3b06f8973540cb8d6705c60d9447cf6c, server=jenkins-hbase3.apache.org,40937,1696250003832 in 183 msec 2023-10-02T12:33:40,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:33:40,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962500140343, region=3b06f8973540cb8d6705c60d9447cf6c, ASSIGN in 345 msec 2023-10-02T12:33:40,987 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962500140343 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:40,987 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962500140343","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250020987"}]},"ts":"1696250020987"} 2023-10-02T12:33:40,989 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962500140343, state=ENABLED in hbase:meta 2023-10-02T12:33:40,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962500140343 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:40,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962500140343 in 849 msec 2023-10-02T12:33:41,261 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:33:41,263 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962500140343, procId: 22 completed 2023-10-02T12:33:41,263 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962500140343 get assigned. Timeout = 60000ms 2023-10-02T12:33:41,263 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:41,290 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962500140343 assigned to meta. Checking AM states. 2023-10-02T12:33:41,290 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:33:41,290 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962500140343 assigned. 2023-10-02T12:33:41,291 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5c379750 to 127.0.0.1:65053 2023-10-02T12:33:41,291 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:41,346 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(147): before: backup.TestBackupDelete#testBackupDeleteCommand Thread=585, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=455, ProcessCount=181, AvailableMemoryMB=2087 2023-10-02T12:33:41,346 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2023-10-02T12:33:41,350 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(86): test backup delete on a single table with data: command-line 2023-10-02T12:33:41,352 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6f926749 to 127.0.0.1:65053 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:41,372 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@862ac60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:41,405 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:41,407 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58324, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:41,420 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:41,422 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:41,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:33:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:33:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:33:41,451 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:33:41,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 21 msec 2023-10-02T12:33:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:33:41,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:33:41,565 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:41,565 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:41,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:33:41,567 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:41,593 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7b374dce01ddb9705e86d3520a03887f, NAME => 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:41,611 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:41,611 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 7b374dce01ddb9705e86d3520a03887f, disabling compactions & flushes 2023-10-02T12:33:41,611 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,611 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,611 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. after waiting 0 ms 2023-10-02T12:33:41,611 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,611 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,611 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:33:41,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:41,614 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250021613"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250021613"}]},"ts":"1696250021613"} 2023-10-02T12:33:41,616 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:41,618 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:41,618 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250021618"}]},"ts":"1696250021618"} 2023-10-02T12:33:41,620 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:33:41,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7b374dce01ddb9705e86d3520a03887f, ASSIGN}] 2023-10-02T12:33:41,628 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7b374dce01ddb9705e86d3520a03887f, ASSIGN 2023-10-02T12:33:41,629 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=7b374dce01ddb9705e86d3520a03887f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:41,781 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7b374dce01ddb9705e86d3520a03887f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:41,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 7b374dce01ddb9705e86d3520a03887f, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:41,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696250014034 2023-10-02T12:33:41,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696250014034 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:41,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962500140341 2023-10-02T12:33:41,793 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962500140341 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:41,794 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962500140342 2023-10-02T12:33:41,794 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962500140342 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:41,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962500140343 2023-10-02T12:33:41,795 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962500140343 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:41,945 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,945 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 7b374dce01ddb9705e86d3520a03887f, NAME => 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:41,946 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:41,946 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,946 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:41,946 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,946 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,958 INFO [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,965 INFO [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b374dce01ddb9705e86d3520a03887f columnFamilyName meta 2023-10-02T12:33:41,965 DEBUG [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:41,966 INFO [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] regionserver.HStore(324): Store=7b374dce01ddb9705e86d3520a03887f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:41,966 INFO [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,969 INFO [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b374dce01ddb9705e86d3520a03887f columnFamilyName session 2023-10-02T12:33:41,969 DEBUG [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:41,970 INFO [StoreOpener-7b374dce01ddb9705e86d3520a03887f-1 {}] regionserver.HStore(324): Store=7b374dce01ddb9705e86d3520a03887f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:41,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,972 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:33:41,977 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:41,981 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:41,982 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 7b374dce01ddb9705e86d3520a03887f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11678319520, jitterRate=0.08762826025485992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:33:41,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:33:41,984 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f., pid=28, masterSystemTime=1696250021937 2023-10-02T12:33:41,986 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,987 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:41,987 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7b374dce01ddb9705e86d3520a03887f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:41,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:33:41,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 7b374dce01ddb9705e86d3520a03887f, server=jenkins-hbase3.apache.org,40937,1696250003832 in 206 msec 2023-10-02T12:33:41,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:33:41,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=7b374dce01ddb9705e86d3520a03887f, ASSIGN in 369 msec 2023-10-02T12:33:41,998 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:41,999 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250021998"}]},"ts":"1696250021998"} 2023-10-02T12:33:42,002 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:33:42,006 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:42,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 447 msec 2023-10-02T12:33:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:33:42,174 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:33:42,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:33:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:33:42,195 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:33:42,196 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:42,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:33:42,197 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:33:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:42,217 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9635224d05e38a05440ef6ab163fbe51, NAME => 'backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:42,235 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:42,235 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 9635224d05e38a05440ef6ab163fbe51, disabling compactions & flushes 2023-10-02T12:33:42,235 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,235 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,235 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. after waiting 0 ms 2023-10-02T12:33:42,235 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,235 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,235 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9635224d05e38a05440ef6ab163fbe51: 2023-10-02T12:33:42,237 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:33:42,237 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696250022237"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250022237"}]},"ts":"1696250022237"} 2023-10-02T12:33:42,240 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:33:42,241 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:33:42,241 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250022241"}]},"ts":"1696250022241"} 2023-10-02T12:33:42,243 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:33:42,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=9635224d05e38a05440ef6ab163fbe51, ASSIGN}] 2023-10-02T12:33:42,250 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=9635224d05e38a05440ef6ab163fbe51, ASSIGN 2023-10-02T12:33:42,251 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=9635224d05e38a05440ef6ab163fbe51, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40937,1696250003832; forceNewPlan=false, retain=false 2023-10-02T12:33:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:42,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=9635224d05e38a05440ef6ab163fbe51, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:42,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 9635224d05e38a05440ef6ab163fbe51, server=jenkins-hbase3.apache.org,40937,1696250003832}] 2023-10-02T12:33:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:42,565 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,565 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 9635224d05e38a05440ef6ab163fbe51, NAME => 'backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:33:42,565 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:33:42,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:33:42,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,569 INFO [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,571 INFO [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9635224d05e38a05440ef6ab163fbe51 columnFamilyName meta 2023-10-02T12:33:42,571 DEBUG [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:42,572 INFO [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] regionserver.HStore(324): Store=9635224d05e38a05440ef6ab163fbe51/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:42,572 INFO [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,574 INFO [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9635224d05e38a05440ef6ab163fbe51 columnFamilyName session 2023-10-02T12:33:42,574 DEBUG [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:33:42,575 INFO [StoreOpener-9635224d05e38a05440ef6ab163fbe51-1 {}] regionserver.HStore(324): Store=9635224d05e38a05440ef6ab163fbe51/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:33:42,577 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system_bulk/9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,578 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system_bulk/9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:33:42,582 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:33:42,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system_bulk/9635224d05e38a05440ef6ab163fbe51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:33:42,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 9635224d05e38a05440ef6ab163fbe51; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9511988480, jitterRate=-0.11412703990936279}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:33:42,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 9635224d05e38a05440ef6ab163fbe51: 2023-10-02T12:33:42,589 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51., pid=31, masterSystemTime=1696250022559 2023-10-02T12:33:42,592 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,592 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:33:42,592 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=9635224d05e38a05440ef6ab163fbe51, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:42,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:33:42,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 9635224d05e38a05440ef6ab163fbe51, server=jenkins-hbase3.apache.org,40937,1696250003832 in 189 msec 2023-10-02T12:33:42,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:33:42,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=9635224d05e38a05440ef6ab163fbe51, ASSIGN in 350 msec 2023-10-02T12:33:42,601 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:33:42,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250022601"}]},"ts":"1696250022601"} 2023-10-02T12:33:42,604 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:33:42,607 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:33:42,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 416 msec 2023-10-02T12:33:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:33:42,806 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:33:42,836 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:33:42,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250022909 (current time:1696250022909). 2023-10-02T12:33:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:33:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1b784da6 to 127.0.0.1:65053 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d59d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:42,929 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58334, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1b784da6 to 127.0.0.1:65053 2023-10-02T12:33:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:33:42,949 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:42,955 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:42,971 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:42,975 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:42,996 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:43,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:33:43,012 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:43,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:33:43,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:43,175 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:33:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:43,267 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/33cc971a8a564ae1b7ab59de6cd15d2a is 49, key is activesession:/session:c/1696250022886/Put/seqid=0 2023-10-02T12:33:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:43,668 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:33:43,669 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962500140342' 2023-10-02T12:33:43,669 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:33:43,669 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962500140343' 2023-10-02T12:33:43,686 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/33cc971a8a564ae1b7ab59de6cd15d2a 2023-10-02T12:33:43,765 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/33cc971a8a564ae1b7ab59de6cd15d2a as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a 2023-10-02T12:33:43,774 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:33:43,779 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 604ms, sequenceid=5, compaction requested=false 2023-10-02T12:33:43,781 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:33:43,781 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:33:43,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:33:43,788 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:43,793 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:33:43,793 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a] hfiles 2023-10-02T12:33:43,793 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:33:43,812 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:43,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:33:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:33:43,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:43,822 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:43,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:33:43,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 823 msec 2023-10-02T12:33:43,826 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:43,828 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:43,831 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:43,832 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:33:43,835 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:33:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:44,262 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:44,277 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:44,277 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:33:44,280 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:44,280 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:33:44,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.3400 sec 2023-10-02T12:33:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:33:45,060 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-10-02T12:33:45,061 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(115): Backup backup_1696250021379 started at 1696250025060. 2023-10-02T12:33:45,094 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(119): Backup session backup_1696250021379 has been started. 2023-10-02T12:33:45,098 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:33:45,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:33:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:33:45,115 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:33:45,116 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:33:45,117 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:33:45,117 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:33:45,117 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,117 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:45,119 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:33:45,119 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,119 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:45,119 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:45,119 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,119 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:33:45,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:45,120 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:33:45,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:45,121 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:33:45,123 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:33:45,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:33:45,123 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:33:45,123 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:33:45,124 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:33:45,124 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:33:45,124 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40937,1696250003832' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:33:45,127 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,127 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:45,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:45,127 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,127 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:33:45,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:45,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:45,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:45,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:45,129 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40937,1696250003832' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:33:45,129 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@45bf92cb[Count = 0] remaining members to acquire global barrier 2023-10-02T12:33:45,129 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:33:45,129 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,136 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,136 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,136 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,137 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:33:45,137 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,137 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:33:45,138 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:33:45,138 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,138 INFO [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250005895 highest: 1696250005895 on jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,138 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832:(num 1696250005895) roll requested 2023-10-02T12:33:45,157 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250025138, exclude list is [], retry=0 2023-10-02T12:33:45,166 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:45,172 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250005895 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250025138 2023-10-02T12:33:45,173 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:45,173 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250005895 is not closed yet, will try archiving it next time 2023-10-02T12:33:45,173 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta:.meta(num 1696250006841) roll requested 2023-10-02T12:33:45,192 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250025173.meta, exclude list is [], retry=0 2023-10-02T12:33:45,196 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:45,199 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250006841.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250025173.meta 2023-10-02T12:33:45,199 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:45,199 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250006841.meta is not closed yet, will try archiving it next time 2023-10-02T12:33:45,202 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 64 2023-10-02T12:33:45,202 INFO [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250025138 on jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,202 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:45,204 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:33:45,225 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:33:45,225 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40937,1696250003832' in zk 2023-10-02T12:33:45,228 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,228 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:33:45,228 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:45,228 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:33:45,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:45,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:45,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:45,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:45,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:45,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:45,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:45,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40937,1696250003832': 2023-10-02T12:33:45,233 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40937,1696250003832' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:33:45,233 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:33:45,233 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:33:45,233 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:33:45,234 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:33:45,234 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:33:45,236 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,236 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,237 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:33:45,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,237 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:45,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:45,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:45,237 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:45,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:45,238 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:45,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:45,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:45,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,240 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:45,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:45,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40937-0x101732a517a0001, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:45,250 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:45,250 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:45,250 DEBUG [Listener at localhost/33177-EventThread {}] zookeeper.ZKWatcher(604): master:37819-0x101732a517a0000, quorum=127.0.0.1:65053, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:45,250 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:45,250 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:45,251 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:45,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:33:45,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:33:45,254 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:33:45,254 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:33:45,354 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:33:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:33:45,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } 2023-10-02T12:33:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250025366 (current time:1696250025366). 2023-10-02T12:33:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250025365_default_test-1696250014034 VERSION not specified, setting to 2 2023-10-02T12:33:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41416b43 to 127.0.0.1:65053 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:33:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8c1274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:45,385 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58344, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41416b43 to 127.0.0.1:65053 2023-10-02T12:33:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } 2023-10-02T12:33:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:33:45,391 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:45,392 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:45,396 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696250025365_default_test-1696250014034 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:45,399 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:45,822 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:45,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e}] 2023-10-02T12:33:45,824 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:45,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-10-02T12:33:45,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:45,979 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 0d134e56945ef6d5e83667f94878fc5e 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:33:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:46,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/.tmp/f/d27a0a44ce614da8b7e96957c8d01838 is 37, key is row10/f:q1/1696250016704/Put/seqid=0 2023-10-02T12:33:46,010 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/.tmp/f/d27a0a44ce614da8b7e96957c8d01838 2023-10-02T12:33:46,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/.tmp/f/d27a0a44ce614da8b7e96957c8d01838 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 2023-10-02T12:33:46,031 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:33:46,033 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 0d134e56945ef6d5e83667f94878fc5e in 53ms, sequenceid=103, compaction requested=false 2023-10-02T12:33:46,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 0d134e56945ef6d5e83667f94878fc5e: 2023-10-02T12:33:46,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. for snapshot_1696250025365_default_test-1696250014034 completed. 2023-10-02T12:33:46,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.' region-info for snapshot=snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:46,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:46,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838] hfiles 2023-10-02T12:33:46,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 for snapshot=snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:46,045 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:46,045 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-10-02T12:33:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-10-02T12:33:46,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250025365_default_test-1696250014034 on region 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:46,046 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:46,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:33:46,049 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:46,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e in 225 msec 2023-10-02T12:33:46,050 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:46,051 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:46,051 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:46,052 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:46,070 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:46,079 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:46,080 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696250025365_default_test-1696250014034 to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:46,081 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:46,082 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:33:46,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250025365_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } in 695 msec 2023-10-02T12:33:46,463 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:33:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:33:46,500 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696250014034, procId: 34 completed 2023-10-02T12:33:46,500 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696250021379 2023-10-02T12:33:46,500 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:33:46,502 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:33:46,505 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250021379_test-1696250014034 2023-10-02T12:33:46,505 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250025365_default_test-1696250014034 to hdfs://localhost:36711/backupUT/backup_1696250021379/default/test-1696250014034/ 2023-10-02T12:33:46,519 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:33:46,555 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36711, inputRoot=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:46,555 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37819,1696250000621_-709021102_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36711/backupUT/backup_1696250021379/default/test-1696250014034, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36711/backupUT/backup_1696250021379/default/test-1696250014034/.hbase-snapshot/.tmp/snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:46,556 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36711, inputRoot=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664, snapshotDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696250025365_default_test-1696250014034. 2023-10-02T12:33:46,581 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696250025365_default_test-1696250014034 to hdfs://localhost:36711/backupUT/backup_1696250021379/default/test-1696250014034/.hbase-snapshot/.tmp/snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:47,087 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:47,088 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:47,089 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:47,090 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,193 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop-1370754494985443518.jar 2023-10-02T12:33:48,194 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,194 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,195 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,195 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,196 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,196 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,197 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,197 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,198 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:48,198 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:48,199 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:33:48,200 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:33:48,200 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:33:48,201 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:33:48,201 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:33:48,201 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:33:48,202 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:33:48,202 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:33:48,203 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:33:48,203 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:33:48,204 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:33:48,208 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:48,208 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:48,209 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:48,209 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:48,210 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:48,210 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:48,211 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:48,274 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:33:48,309 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250025365_default_test-1696250014034' hfile list 2023-10-02T12:33:48,323 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:33:48,600 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir/1696250028534/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:33:48,600 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir/1696250028534/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:33:48,832 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:33:48,866 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 output=hdfs://localhost:36711/backupUT/backup_1696250021379/default/test-1696250014034/archive/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 2023-10-02T12:33:48,867 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:33:49,687 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:33:49,689 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:33:49,697 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:49,697 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250025365_default_test-1696250014034 finished. 2023-10-02T12:33:49,698 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1034): test-1696250014034 2023-10-02T12:33:49,781 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250021379 2023-10-02T12:33:49,781 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:33:49,804 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36711/backupUT/backup_1696250021379/.backup.manifest 2023-10-02T12:33:49,805 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(358): Backup backup_1696250021379 finished: type=FULL,tablelist=test-1696250014034,targetRootDir=hdfs://localhost:36711/backupUT,startts=1696250025060,completets=1696250029778,bytescopied=0 2023-10-02T12:33:49,805 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:33:49,805 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:49,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250025365_default_test-1696250014034" 2023-10-02T12:33:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250025365_default_test-1696250014034 2023-10-02T12:33:49,819 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250025365_default_test-1696250014034 for backup backup_1696250021379 succeeded. 2023-10-02T12:33:49,820 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:33:49,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:33:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:33:49,840 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:33:49,848 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:33:49,871 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(376): Backup backup_1696250021379 completed. 2023-10-02T12:33:49,871 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:33:49,872 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6f926749 to 127.0.0.1:65053 2023-10-02T12:33:49,872 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:49,913 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(90): backup complete 2023-10-02T12:33:49,943 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608b29cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:49,944 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:49,946 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55580, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:49,952 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:49,954 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:49,984 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:33:49,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250029994 (current time:1696250029994). 2023-10-02T12:33:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:33:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2997d989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:50,007 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58370, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:33:50,014 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:50,015 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:50,019 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:50,023 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:50,061 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:50,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:33:50,062 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:50,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-10-02T12:33:50,214 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:50,215 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=1.14 KB heapSize=2.37 KB 2023-10-02T12:33:50,231 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/6a130b58b1dd41faaf001a587b4eaf06 is 169, key is trslm:hdfs://localhost:36711/backupUT\x00test-1696250014034/meta:log-roll-map/1696250029766/Put/seqid=0 2023-10-02T12:33:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:50,642 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=586 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/6a130b58b1dd41faaf001a587b4eaf06 2023-10-02T12:33:50,654 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a4a1542146ae4ced9e686a2e847de665 is 310, key is session:backup_1696250021379/session:context/1696250029846/Put/seqid=0 2023-10-02T12:33:51,067 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a4a1542146ae4ced9e686a2e847de665 2023-10-02T12:33:51,076 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/6a130b58b1dd41faaf001a587b4eaf06 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 2023-10-02T12:33:51,084 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06, entries=5, sequenceid=18, filesize=5.5 K 2023-10-02T12:33:51,094 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a4a1542146ae4ced9e686a2e847de665 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 2023-10-02T12:33:51,103 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665, entries=2, sequenceid=18, filesize=5.3 K 2023-10-02T12:33:51,104 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1165, heapSize ~2.34 KB/2392, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 889ms, sequenceid=18, compaction requested=false 2023-10-02T12:33:51,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:33:51,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:33:51,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:33:51,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:51,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06] hfiles 2023-10-02T12:33:51,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 for snapshot=snapshot_backup_system 2023-10-02T12:33:51,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665] hfiles 2023-10-02T12:33:51,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:33:51,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 for snapshot=snapshot_backup_system 2023-10-02T12:33:51,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:51,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-10-02T12:33:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=37 2023-10-02T12:33:51,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:51,124 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:51,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-10-02T12:33:51,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 1.0640 sec 2023-10-02T12:33:51,129 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:51,130 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:51,131 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:51,131 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:33:51,132 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:33:51,164 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:51,194 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:51,195 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:33:51,197 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:51,197 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:33:51,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1870 sec 2023-10-02T12:33:51,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:33:51,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:51,792 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:33:51,792 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:33:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:33:52,139 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696250021379 ... 2023-10-02T12:33:52,142 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:33:52,143 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:33:52,144 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696250021379 2023-10-02T12:33:52,145 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36711/backupUT/backup_1696250021379/default/test-1696250014034. 2023-10-02T12:33:52,146 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(465): hdfs://localhost:36711/backupUT/backup_1696250021379/default is empty, remove it. 2023-10-02T12:33:52,157 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:33:52,157 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:33:52,162 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(276): Delete backup backup_1696250021379 completed. 2023-10-02T12:33:52,172 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1034): test-1696250014034 2023-10-02T12:33:52,176 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:33:52,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:33:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:33:52,187 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:33:52,188 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:33:52,192 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:52,192 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(103): delete_backup 2023-10-02T12:33:52,192 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(105): Deleted 1 backups. Total requested: 1 2023-10-02T12:33:52,219 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(175): after: backup.TestBackupDelete#testBackupDeleteCommand Thread=599 (was 585) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1520423817_16 at /127.0.0.1:60032 [Receiving block BP-1566406829-172.31.12.81-1696249997240:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:65053@0x23dc0da4 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$496/1312974157.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1566406829-172.31.12.81-1696249997240:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:60080 [Waiting for operation #16] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1520423817_16 at /127.0.0.1:60048 [Receiving block BP-1566406829-172.31.12.81-1696249997240:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:60092 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:65053@0x23dc0da4-SendThread(127.0.0.1:65053) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_203573507_16 at /127.0.0.1:52788 [Waiting for operation #10] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:65053@0x23dc0da4-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-4 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1566406829-172.31.12.81-1696249997240:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-metaLookup-shared--pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=682 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=513 (was 455) - SystemLoadAverage LEAK? -, ProcessCount=184 (was 181) - ProcessCount LEAK? -, AvailableMemoryMB=1444 (was 2087) 2023-10-02T12:33:52,219 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(130): Thread=599 is superior to 500 2023-10-02T12:33:52,241 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(147): before: backup.TestBackupDelete#testBackupDelete Thread=599, OpenFileDescriptor=682, MaxFileDescriptor=60000, SystemLoadAverage=513, ProcessCount=184, AvailableMemoryMB=1438 2023-10-02T12:33:52,241 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(130): Thread=599 is superior to 500 2023-10-02T12:33:52,241 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(59): test backup delete on a single table with data 2023-10-02T12:33:52,250 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@114b5075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:52,257 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:52,263 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:52,266 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:33:52,267 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33946, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:33:52,288 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:33:52,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250032295 (current time:1696250032295). 2023-10-02T12:33:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:33:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be840cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:52,317 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51158, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-10-02T12:33:52,323 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:33:52,325 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:52,337 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:52,341 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:52,380 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:52,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:33:52,383 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:33:52,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=39 2023-10-02T12:33:52,535 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:52,536 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-10-02T12:33:52,541 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/c23235ae4a724b73a1da50ce890f08f0 is 91, key is incrbackupset:hdfs://localhost:36711/backupUT/meta:test-1696250014034/1696250032173/Put/seqid=0 2023-10-02T12:33:52,547 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/c23235ae4a724b73a1da50ce890f08f0 2023-10-02T12:33:52,554 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c23235ae4a724b73a1da50ce890f08f0 2023-10-02T12:33:52,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a8885f890ad944f08664ca3b511fbcc0 is 59, key is session:backup_1696250021379/session:/1696250032161/DeleteFamily/seqid=0 2023-10-02T12:33:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:33:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:33:52,970 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a8885f890ad944f08664ca3b511fbcc0 2023-10-02T12:33:52,978 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a8885f890ad944f08664ca3b511fbcc0 2023-10-02T12:33:52,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/c23235ae4a724b73a1da50ce890f08f0 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0 2023-10-02T12:33:52,987 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c23235ae4a724b73a1da50ce890f08f0 2023-10-02T12:33:52,987 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0, entries=2, sequenceid=26, filesize=5.2 K 2023-10-02T12:33:52,988 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a8885f890ad944f08664ca3b511fbcc0 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0 2023-10-02T12:33:52,995 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a8885f890ad944f08664ca3b511fbcc0 2023-10-02T12:33:52,995 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0, entries=2, sequenceid=26, filesize=5.2 K 2023-10-02T12:33:52,997 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 461ms, sequenceid=26, compaction requested=true 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0] hfiles 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 for snapshot=snapshot_backup_system 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0 for snapshot=snapshot_backup_system 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0] hfiles 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 for snapshot=snapshot_backup_system 2023-10-02T12:33:52,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0 for snapshot=snapshot_backup_system 2023-10-02T12:33:53,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:53,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=39 2023-10-02T12:33:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=39 2023-10-02T12:33:53,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:53,415 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:53,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-10-02T12:33:53,424 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:53,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 1.0350 sec 2023-10-02T12:33:53,425 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:53,426 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:53,426 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:33:53,427 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:33:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:33:53,466 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:53,477 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:53,478 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:33:53,480 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:53,480 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-10-02T12:33:53,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1610 sec 2023-10-02T12:33:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:33:54,432 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(115): Backup backup_1696250032251 started at 1696250034432. 2023-10-02T12:33:54,435 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(119): Backup session backup_1696250032251 has been started. 2023-10-02T12:33:54,437 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:33:54,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:33:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:33:54,439 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:33:54,439 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:33:54,441 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:33:54,441 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:33:54,441 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:54,443 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,443 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:54,443 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:54,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:54,444 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:33:54,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:33:54,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:33:54,445 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:33:54,445 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:33:54,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:33:54,449 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:33:54,449 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:33:54,450 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:33:54,450 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:33:54,450 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40937,1696250003832' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:33:54,454 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:54,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:54,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:54,455 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:33:54,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:54,458 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:54,458 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:54,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:54,460 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40937,1696250003832' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:33:54,460 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7da2f583[Count = 0] remaining members to acquire global barrier 2023-10-02T12:33:54,460 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:33:54,460 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:54,461 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:54,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:33:54,461 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:33:54,462 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:33:54,462 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:33:54,462 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,462 INFO [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250025138 highest: 1696250025138 on jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,463 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832:(num 1696250025138) roll requested 2023-10-02T12:33:54,480 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250034463, exclude list is [], retry=0 2023-10-02T12:33:54,486 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:54,490 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250025138 with entries=20, filesize=3.96 KB; new WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250034463 2023-10-02T12:33:54,491 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:54,491 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250025138 is not closed yet, will try archiving it next time 2023-10-02T12:33:54,491 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta:.meta(num 1696250025173) roll requested 2023-10-02T12:33:54,509 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034491.meta, exclude list is [], retry=0 2023-10-02T12:33:54,517 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:33:54,522 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250025173.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034491.meta 2023-10-02T12:33:54,522 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:33:54,522 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250025173.meta is not closed yet, will try archiving it next time 2023-10-02T12:33:54,523 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:33:54,523 INFO [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250034463 on jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,530 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250025173.meta to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250025173.meta 2023-10-02T12:33:54,550 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:33:54,551 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40937,1696250003832' in zk 2023-10-02T12:33:54,554 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:33:54,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,554 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:54,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:54,554 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:33:54,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:54,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:54,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:54,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:54,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:54,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:54,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40937,1696250003832': 2023-10-02T12:33:54,560 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40937,1696250003832' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:33:54,560 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:33:54,560 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:33:54,560 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:33:54,560 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:33:54,560 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:33:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:54,563 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:33:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:33:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:33:54,563 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:33:54,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:54,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:33:54,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:54,565 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:33:54,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:54,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,567 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:33:54,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:33:54,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:33:54,578 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:33:54,578 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:33:54,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:33:54,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:33:54,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:33:54,580 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:33:54,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:33:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:33:54,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } 2023-10-02T12:33:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250034686 (current time:1696250034686). 2023-10-02T12:33:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250034684_default_test-1696250014034 VERSION not specified, setting to 2 2023-10-02T12:33:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce9acc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:54,698 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51170, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=40, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } 2023-10-02T12:33:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-10-02T12:33:54,704 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:33:54,705 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:54,708 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696250034684_default_test-1696250014034 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:54,711 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:33:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:33:55,119 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:55,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e}] 2023-10-02T12:33:55,121 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:55,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=41 2023-10-02T12:33:55,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:55,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2520): Flush status journal for 0d134e56945ef6d5e83667f94878fc5e: 2023-10-02T12:33:55,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. for snapshot_1696250034684_default_test-1696250014034 completed. 2023-10-02T12:33:55,274 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(240): Storing 'test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.' region-info for snapshot=snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:55,274 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:55,274 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838] hfiles 2023-10-02T12:33:55,274 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 for snapshot=snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:55,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:33:55,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=41 2023-10-02T12:33:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=41 2023-10-02T12:33:55,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250034684_default_test-1696250014034 on region 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:55,282 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:33:55,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-10-02T12:33:55,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e in 164 msec 2023-10-02T12:33:55,285 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:55,286 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:55,287 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:55,287 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:55,288 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:33:55,698 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:55,705 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:55,706 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696250034684_default_test-1696250014034 to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:55,708 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:55,708 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-10-02T12:33:55,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_1696250034684_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } in 1.0080 sec 2023-10-02T12:33:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:33:55,810 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696250032251 2023-10-02T12:33:55,811 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:33:55,811 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:33:55,811 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250032251_test-1696250014034 2023-10-02T12:33:55,811 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250034684_default_test-1696250014034 to hdfs://localhost:36711/backupUT/backup_1696250032251/default/test-1696250014034/ 2023-10-02T12:33:55,811 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:33:55,835 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36711, inputRoot=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:33:55,835 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37819,1696250000621_-690513217_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36711/backupUT/backup_1696250032251/default/test-1696250014034, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36711/backupUT/backup_1696250032251/default/test-1696250014034/.hbase-snapshot/.tmp/snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:55,835 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36711, inputRoot=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664, snapshotDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696250034684_default_test-1696250014034. 2023-10-02T12:33:55,845 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696250034684_default_test-1696250014034 to hdfs://localhost:36711/backupUT/backup_1696250032251/default/test-1696250014034/.hbase-snapshot/.tmp/snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:56,260 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:56,261 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:56,262 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:56,262 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,352 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop-9029602227263305306.jar 2023-10-02T12:33:57,352 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,353 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,353 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,354 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,354 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,354 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,355 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,355 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,356 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:33:57,356 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:33:57,357 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:33:57,357 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:33:57,358 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:33:57,359 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:33:57,360 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:33:57,360 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:33:57,361 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:33:57,362 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:33:57,362 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:33:57,363 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:33:57,363 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:33:57,364 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:57,364 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:57,365 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:57,365 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:57,365 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:33:57,366 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:57,367 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:33:57,416 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250034684_default_test-1696250014034' hfile list 2023-10-02T12:33:57,419 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:33:57,508 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir/1696250037462/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:33:57,508 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir/1696250037462/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:33:57,579 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:33:57,642 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 output=hdfs://localhost:36711/backupUT/backup_1696250032251/default/test-1696250014034/archive/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 2023-10-02T12:33:57,642 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:33:58,312 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:33:58,314 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33956, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:33:58,558 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:33:58,559 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:33:58,570 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:58,570 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250034684_default_test-1696250014034 finished. 2023-10-02T12:33:58,570 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1034): test-1696250014034 2023-10-02T12:33:58,581 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250032251 2023-10-02T12:33:58,582 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:33:58,589 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36711/backupUT/backup_1696250032251/.backup.manifest 2023-10-02T12:33:58,589 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(358): Backup backup_1696250032251 finished: type=FULL,tablelist=test-1696250014034,targetRootDir=hdfs://localhost:36711/backupUT,startts=1696250034432,completets=1696250038581,bytescopied=0 2023-10-02T12:33:58,589 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:33:58,590 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:58,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250034684_default_test-1696250014034" 2023-10-02T12:33:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250034684_default_test-1696250014034 2023-10-02T12:33:58,598 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250034684_default_test-1696250014034 for backup backup_1696250032251 succeeded. 2023-10-02T12:33:58,599 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:33:58,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:33:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:33:58,613 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:33:58,617 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:33:58,627 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(376): Backup backup_1696250032251 completed. 2023-10-02T12:33:58,627 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:58,648 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(63): backup complete 2023-10-02T12:33:58,684 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:33:58,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250038698 (current time:1696250038698). 2023-10-02T12:33:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:33:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:33:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc3001a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:33:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:33:58,736 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51176, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:33:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:33:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:33:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:33:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:33:58,746 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:33:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:58,747 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:33:58,752 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:33:58,755 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:33:58,788 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:33:58,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:33:58,791 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:58,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-10-02T12:33:58,943 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:58,943 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=1.06 KB heapSize=2.22 KB 2023-10-02T12:33:58,949 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/d51e289aac0b41dc99d6206ac83a9648 is 169, key is trslm:hdfs://localhost:36711/backupUT\x00test-1696250014034/meta:log-roll-map/1696250038574/Put/seqid=0 2023-10-02T12:33:58,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=511 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/d51e289aac0b41dc99d6206ac83a9648 2023-10-02T12:33:58,975 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/9e728ac0551c4adfae6a5a347024e85f is 310, key is session:backup_1696250032251/session:context/1696250038614/Put/seqid=0 2023-10-02T12:33:58,987 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/9e728ac0551c4adfae6a5a347024e85f 2023-10-02T12:33:58,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/d51e289aac0b41dc99d6206ac83a9648 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648 2023-10-02T12:33:59,011 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648, entries=5, sequenceid=38, filesize=5.5 K 2023-10-02T12:33:59,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/9e728ac0551c4adfae6a5a347024e85f as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f 2023-10-02T12:33:59,026 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f, entries=2, sequenceid=38, filesize=5.3 K 2023-10-02T12:33:59,027 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~1.06 KB/1090, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 84ms, sequenceid=38, compaction requested=true 2023-10-02T12:33:59,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:33:59,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648] hfiles 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0 for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648 for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f] hfiles 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0 for snapshot=snapshot_backup_system 2023-10-02T12:33:59,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f for snapshot=snapshot_backup_system 2023-10-02T12:33:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:59,439 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:33:59,439 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-10-02T12:33:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-10-02T12:33:59,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:59,441 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:33:59,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-10-02T12:33:59,444 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:33:59,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 654 msec 2023-10-02T12:33:59,445 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:33:59,446 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:33:59,446 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:33:59,447 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:33:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:33:59,859 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:33:59,869 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:33:59,869 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:33:59,870 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:33:59,871 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:33:59,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1330 sec 2023-10-02T12:34:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:34:00,862 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696250032251 ... 2023-10-02T12:34:00,863 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:34:00,864 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:34:00,865 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696250032251 2023-10-02T12:34:00,866 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36711/backupUT/backup_1696250032251/default/test-1696250014034. 2023-10-02T12:34:00,867 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(465): hdfs://localhost:36711/backupUT/backup_1696250032251/default is empty, remove it. 2023-10-02T12:34:00,872 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:34:00,872 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:34:00,874 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(276): Delete backup backup_1696250032251 completed. 2023-10-02T12:34:00,883 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1034): test-1696250014034 2023-10-02T12:34:00,886 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:34:00,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:34:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:34:00,897 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:34:00,897 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:34:00,901 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(76): delete_backup 2023-10-02T12:34:00,922 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(175): after: backup.TestBackupDelete#testBackupDelete Thread=602 (was 599) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-metaLookup-shared--pool-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1520423817_16 at /127.0.0.1:52864 [Receiving block BP-1566406829-172.31.12.81-1696249997240:blk_1073741875_1051] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1520423817_16 at /127.0.0.1:52876 [Receiving block BP-1566406829-172.31.12.81-1696249997240:blk_1073741876_1052] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_203573507_16 at /127.0.0.1:53016 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:52844 [Waiting for operation #19] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1566406829-172.31.12.81-1696249997240:blk_1073741876_1052, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:52926 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1566406829-172.31.12.81-1696249997240:blk_1073741875_1051, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-8 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-5 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=683 (was 682) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=512 (was 513), ProcessCount=184 (was 184), AvailableMemoryMB=1230 (was 1438) 2023-10-02T12:34:00,922 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-10-02T12:34:00,939 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(147): before: backup.TestBackupDelete#testBackupPurgeOldBackupsCommand Thread=602, OpenFileDescriptor=683, MaxFileDescriptor=60000, SystemLoadAverage=512, ProcessCount=184, AvailableMemoryMB=1229 2023-10-02T12:34:00,939 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-10-02T12:34:00,940 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(111): test backup delete (purge old backups) on a single table with data: command-line 2023-10-02T12:34:00,951 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51922dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:00,954 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:00,956 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51350, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:00,958 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:00,960 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57510, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:00,977 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:34:00,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696077240984 (current time:1696077240984). 2023-10-02T12:34:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:34:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76feaf94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:00,996 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-10-02T12:34:01,002 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:01,003 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:34:01,007 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:01,009 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:01,019 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:01,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:34:01,021 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:34:01,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=45 2023-10-02T12:34:01,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:01,173 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-10-02T12:34:01,178 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/e1812de71f564ba89763784fb63a53bb is 91, key is incrbackupset:hdfs://localhost:36711/backupUT/meta:test-1696250014034/1696250040884/Put/seqid=0 2023-10-02T12:34:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:34:01,583 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/e1812de71f564ba89763784fb63a53bb 2023-10-02T12:34:01,590 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e1812de71f564ba89763784fb63a53bb 2023-10-02T12:34:01,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/ad264bb5ce304d6287d6ecfa585c14d6 is 59, key is session:backup_1696250032251/session:/1696250040873/DeleteFamily/seqid=0 2023-10-02T12:34:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:34:02,000 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/ad264bb5ce304d6287d6ecfa585c14d6 2023-10-02T12:34:02,006 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ad264bb5ce304d6287d6ecfa585c14d6 2023-10-02T12:34:02,009 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/e1812de71f564ba89763784fb63a53bb as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb 2023-10-02T12:34:02,021 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e1812de71f564ba89763784fb63a53bb 2023-10-02T12:34:02,021 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb, entries=2, sequenceid=46, filesize=5.2 K 2023-10-02T12:34:02,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/ad264bb5ce304d6287d6ecfa585c14d6 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6 2023-10-02T12:34:02,038 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ad264bb5ce304d6287d6ecfa585c14d6 2023-10-02T12:34:02,038 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6, entries=2, sequenceid=46, filesize=5.2 K 2023-10-02T12:34:02,040 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 867ms, sequenceid=46, compaction requested=true 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb] hfiles 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0 for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648 for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6] hfiles 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 for snapshot=snapshot_backup_system 2023-10-02T12:34:02,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0 for snapshot=snapshot_backup_system 2023-10-02T12:34:02,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f for snapshot=snapshot_backup_system 2023-10-02T12:34:02,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6 for snapshot=snapshot_backup_system 2023-10-02T12:34:02,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:02,058 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-10-02T12:34:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=45 2023-10-02T12:34:02,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:02,060 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:02,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-10-02T12:34:02,064 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:02,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 1.0420 sec 2023-10-02T12:34:02,065 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:02,066 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:02,066 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:34:02,067 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:34:02,080 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:02,096 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:02,096 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:34:02,098 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:02,098 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-10-02T12:34:02,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0990 sec 2023-10-02T12:34:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:34:02,112 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(115): Backup backup_1696077240951 started at 1696077242112. 2023-10-02T12:34:02,115 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(119): Backup session backup_1696077240951 has been started. 2023-10-02T12:34:02,118 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:34:02,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:34:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:34:02,121 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:34:02,121 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:34:02,122 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:34:02,122 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:34:02,123 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:02,136 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,137 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:34:02,138 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:34:02,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:02,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:02,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:34:02,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:02,140 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:34:02,140 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:34:02,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:34:02,140 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:34:02,140 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:34:02,142 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:34:02,142 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:34:02,142 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40937,1696250003832' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:34:02,144 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:02,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:02,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:02,144 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:34:02,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:02,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:02,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:02,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:02,151 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40937,1696250003832' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:34:02,151 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@525f23dd[Count = 0] remaining members to acquire global barrier 2023-10-02T12:34:02,151 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:34:02,151 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:02,152 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:02,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:02,153 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:34:02,153 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:34:02,153 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:34:02,153 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,153 INFO [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250034463 highest: 1696250034463 on jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,153 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832:(num 1696250034463) roll requested 2023-10-02T12:34:02,171 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250034464, exclude list is [], retry=0 2023-10-02T12:34:02,177 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:34:02,180 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250034463 with entries=18, filesize=3.44 KB; new WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250034464 2023-10-02T12:34:02,180 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:34:02,180 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250034463 is not closed yet, will try archiving it next time 2023-10-02T12:34:02,182 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta:.meta(num 1696250034491) roll requested 2023-10-02T12:34:02,194 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250025138 to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs/jenkins-hbase3.apache.org%2C40937%2C1696250003832.1696250025138 2023-10-02T12:34:02,202 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034492.meta, exclude list is [], retry=0 2023-10-02T12:34:02,208 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK] 2023-10-02T12:34:02,210 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034491.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034492.meta 2023-10-02T12:34:02,210 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:44529,DS-feec64fc-1707-4319-84a0-afbe06c78959,DISK]] 2023-10-02T12:34:02,210 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034491.meta is not closed yet, will try archiving it next time 2023-10-02T12:34:02,214 DEBUG [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:34:02,214 INFO [rs(jenkins-hbase3.apache.org,40937,1696250003832)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250034464 on jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,215 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/WALs/jenkins-hbase3.apache.org,40937,1696250003832/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034491.meta to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs/jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta.1696250034491.meta 2023-10-02T12:34:02,228 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:34:02,228 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40937,1696250003832' in zk 2023-10-02T12:34:02,231 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:34:02,231 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:34:02,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,231 DEBUG [member: 'jenkins-hbase3.apache.org,40937,1696250003832' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:34:02,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:02,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:02,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:02,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:02,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:02,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:02,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:02,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40937,1696250003832': 2023-10-02T12:34:02,237 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40937,1696250003832' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:34:02,237 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:34:02,237 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:34:02,237 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:34:02,237 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:34:02,237 INFO [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:34:02,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:02,239 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:02,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:02,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:02,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:02,240 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:34:02,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:02,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:02,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:02,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:02,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:02,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:02,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:02,244 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:02,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:02,255 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:34:02,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:02,256 DEBUG [(jenkins-hbase3.apache.org,37819,1696250000621)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:34:02,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:34:02,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:34:02,257 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:34:02,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:34:02,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } 2023-10-02T12:34:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696077242364 (current time:1696077242364). 2023-10-02T12:34:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696077242363_default_test-1696250014034 VERSION not specified, setting to 2 2023-10-02T12:34:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64085fc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:02,381 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51372, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } 2023-10-02T12:34:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-10-02T12:34:02,387 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:34:02,388 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:02,392 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696077242363_default_test-1696250014034 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:02,394 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:02,404 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:02,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e}] 2023-10-02T12:34:02,406 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:34:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:34:02,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=47 2023-10-02T12:34:02,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:02,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for 0d134e56945ef6d5e83667f94878fc5e: 2023-10-02T12:34:02,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. for snapshot_1696077242363_default_test-1696250014034 completed. 2023-10-02T12:34:02,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(240): Storing 'test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e.' region-info for snapshot=snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:02,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:02,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838] hfiles 2023-10-02T12:34:02,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 for snapshot=snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:02,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:02,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-10-02T12:34:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=47 2023-10-02T12:34:02,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696077242363_default_test-1696250014034 on region 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:34:02,631 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:34:02,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-10-02T12:34:02,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; SnapshotRegionProcedure 0d134e56945ef6d5e83667f94878fc5e in 228 msec 2023-10-02T12:34:02,636 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:02,637 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:02,638 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:02,638 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:02,639 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:02,677 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:02,686 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:02,687 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_1696077242363_default_test-1696250014034 to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:02,689 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:02,689 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-10-02T12:34:02,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_1696077242363_default_test-1696250014034 table=test-1696250014034 type=FLUSH ttl=0 } in 306 msec 2023-10-02T12:34:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:34:02,692 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696077240951 2023-10-02T12:34:02,692 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:34:02,692 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:34:02,692 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696077240951_test-1696250014034 2023-10-02T12:34:02,693 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696077242363_default_test-1696250014034 to hdfs://localhost:36711/backupUT/backup_1696077240951/default/test-1696250014034/ 2023-10-02T12:34:02,693 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:34:02,729 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36711, inputRoot=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664 2023-10-02T12:34:02,729 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37819,1696250000621_379084474_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36711/backupUT/backup_1696077240951/default/test-1696250014034, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36711/backupUT/backup_1696077240951/default/test-1696250014034/.hbase-snapshot/.tmp/snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:02,729 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36711, inputRoot=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664, snapshotDir=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696077242363_default_test-1696250014034. 2023-10-02T12:34:02,751 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_1696077242363_default_test-1696250014034 to hdfs://localhost:36711/backupUT/backup_1696077240951/default/test-1696250014034/.hbase-snapshot/.tmp/snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:03,205 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:03,206 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:03,206 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:03,207 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,535 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop-7740306403229179264.jar 2023-10-02T12:34:04,535 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,536 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,536 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,537 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,537 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,538 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,538 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,539 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,540 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:04,541 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:34:04,541 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:34:04,542 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:34:04,542 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:34:04,543 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:34:04,543 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:34:04,544 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:34:04,544 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:34:04,544 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:34:04,545 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:34:04,545 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:34:04,546 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:34:04,546 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:04,547 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:04,548 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:04,548 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:04,548 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:04,549 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:04,550 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:04,594 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696077242363_default_test-1696250014034' hfile list 2023-10-02T12:34:04,597 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:34:04,689 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir/1696250044647/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:34:04,690 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/mapreduce.cluster.local.dir/1696250044647/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:34:04,759 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:34:04,787 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 output=hdfs://localhost:36711/backupUT/backup_1696077240951/default/test-1696250014034/archive/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/f/d27a0a44ce614da8b7e96957c8d01838 2023-10-02T12:34:04,787 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 2.669M/sec 2023-10-02T12:34:05,734 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:34:05,735 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:34:05,752 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:05,752 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696077242363_default_test-1696250014034 finished. 2023-10-02T12:34:05,752 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1034): test-1696250014034 2023-10-02T12:34:05,762 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696077240951 2023-10-02T12:34:05,762 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:34:05,768 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36711/backupUT/backup_1696077240951/.backup.manifest 2023-10-02T12:34:05,768 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(358): Backup backup_1696077240951 finished: type=FULL,tablelist=test-1696250014034,targetRootDir=hdfs://localhost:36711/backupUT,startts=1696077242112,completets=1696077245762,bytescopied=0 2023-10-02T12:34:05,768 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:34:05,768 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:05,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696077242363_default_test-1696250014034" 2023-10-02T12:34:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696077242363_default_test-1696250014034 2023-10-02T12:34:05,774 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696077242363_default_test-1696250014034 for backup backup_1696077240951 succeeded. 2023-10-02T12:34:05,775 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:34:05,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:34:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:34:05,798 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:34:05,801 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:34:05,808 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.TableBackupClient(376): Backup backup_1696077240951 completed. 2023-10-02T12:34:05,809 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:05,831 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(125): backup complete 2023-10-02T12:34:05,847 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e87f2ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:05,848 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:05,850 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57516, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:05,854 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:05,857 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51384, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:05,921 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:34:05,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250045940 (current time:1696250045940). 2023-10-02T12:34:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:34:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@571647e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:05,962 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-10-02T12:34:05,967 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:34:05,969 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:05,972 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:05,975 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:05,993 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:05,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:34:05,994 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:34:06,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2023-10-02T12:34:06,147 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:06,147 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=1.04 KB heapSize=2.20 KB 2023-10-02T12:34:06,153 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/6615826667344b75aa671a92f29821b3 is 169, key is trslm:hdfs://localhost:36711/backupUT\x00test-1696250014034/meta:log-roll-map/1696077245756/Put/seqid=0 2023-10-02T12:34:06,160 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=491 B at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/6615826667344b75aa671a92f29821b3 2023-10-02T12:34:06,171 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/17639d0f9bc342debec2d51b8f46eacb is 310, key is session:backup_1696077240951/session:context/1696077245799/Put/seqid=0 2023-10-02T12:34:06,175 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/17639d0f9bc342debec2d51b8f46eacb 2023-10-02T12:34:06,182 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/6615826667344b75aa671a92f29821b3 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6615826667344b75aa671a92f29821b3 2023-10-02T12:34:06,188 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6615826667344b75aa671a92f29821b3, entries=5, sequenceid=58, filesize=5.5 K 2023-10-02T12:34:06,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/17639d0f9bc342debec2d51b8f46eacb as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/17639d0f9bc342debec2d51b8f46eacb 2023-10-02T12:34:06,196 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/17639d0f9bc342debec2d51b8f46eacb, entries=2, sequenceid=58, filesize=5.3 K 2023-10-02T12:34:06,197 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3022): Finished flush of dataSize ~1.04 KB/1070, heapSize ~2.16 KB/2216, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 50ms, sequenceid=58, compaction requested=true 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6615826667344b75aa671a92f29821b3] hfiles 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6615826667344b75aa671a92f29821b3 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/17639d0f9bc342debec2d51b8f46eacb] hfiles 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6 for snapshot=snapshot_backup_system 2023-10-02T12:34:06,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/17639d0f9bc342debec2d51b8f46eacb for snapshot=snapshot_backup_system 2023-10-02T12:34:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:34:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:34:06,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:06,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-10-02T12:34:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=49 2023-10-02T12:34:06,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:06,608 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:06,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-10-02T12:34:06,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 616 msec 2023-10-02T12:34:06,611 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:06,612 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:06,613 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:06,613 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:34:06,614 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:34:06,740 INFO [regionserver/jenkins-hbase3:0.Chore.2 {}] hbase.ScheduledChore(145): Chore: MemstoreFlusherChore missed its start time 2023-10-02T12:34:06,740 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(145): Chore: CompactionChecker missed its start time 2023-10-02T12:34:07,032 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:07,046 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:07,047 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:34:07,048 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:07,048 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-10-02T12:34:07,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0840 sec 2023-10-02T12:34:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:34:07,079 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:34:07,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:34:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:34:07,088 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:34:07,088 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:34:07,096 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:07,096 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(142): Deleted 0 backups. Total older than 3 days: 0 2023-10-02T12:34:07,108 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50230df2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:07,110 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:07,112 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57532, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:07,116 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:07,117 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51392, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:07,156 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:34:07,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250047166 (current time:1696250047166). 2023-10-02T12:34:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:34:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742c5406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:07,177 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51406, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-10-02T12:34:07,185 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:07,186 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:07,189 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot, current working dir path: hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:07,191 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:07,200 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:07,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f}] 2023-10-02T12:34:07,202 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:07,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40937 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-10-02T12:34:07,354 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:07,354 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=186 B heapSize=1000 B 2023-10-02T12:34:07,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6 is 64, key is delete_op_row/meta:fam/1696250047163/Put/seqid=0 2023-10-02T12:34:07,368 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=97 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6 2023-10-02T12:34:07,376 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4bb46055ae2a4c41a10e7cfb4d7b90e6 2023-10-02T12:34:07,380 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a812fd6162a940c689e7a5ffc3b35db2 is 49, key is activesession:/session:c/1696250047159/Put/seqid=0 2023-10-02T12:34:07,385 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=89 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a812fd6162a940c689e7a5ffc3b35db2 2023-10-02T12:34:07,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6 2023-10-02T12:34:07,398 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4bb46055ae2a4c41a10e7cfb4d7b90e6 2023-10-02T12:34:07,399 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6, entries=2, sequenceid=65, filesize=5.1 K 2023-10-02T12:34:07,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/a812fd6162a940c689e7a5ffc3b35db2 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a812fd6162a940c689e7a5ffc3b35db2 2023-10-02T12:34:07,406 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a812fd6162a940c689e7a5ffc3b35db2, entries=1, sequenceid=65, filesize=4.9 K 2023-10-02T12:34:07,407 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~186 B/186, heapSize ~968 B/968, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 53ms, sequenceid=65, compaction requested=true 2023-10-02T12:34:07,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:34:07,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. for snapshot_backup_system completed. 2023-10-02T12:34:07,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6615826667344b75aa671a92f29821b3, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6] hfiles 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6a130b58b1dd41faaf001a587b4eaf06 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/c23235ae4a724b73a1da50ce890f08f0 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/d51e289aac0b41dc99d6206ac83a9648 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/e1812de71f564ba89763784fb63a53bb for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/6615826667344b75aa671a92f29821b3 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/4bb46055ae2a4c41a10e7cfb4d7b90e6 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/17639d0f9bc342debec2d51b8f46eacb, hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a812fd6162a940c689e7a5ffc3b35db2] hfiles 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/33cc971a8a564ae1b7ab59de6cd15d2a for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a4a1542146ae4ced9e686a2e847de665 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a8885f890ad944f08664ca3b511fbcc0 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/9e728ac0551c4adfae6a5a347024e85f for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/ad264bb5ce304d6287d6ecfa585c14d6 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/17639d0f9bc342debec2d51b8f46eacb for snapshot=snapshot_backup_system 2023-10-02T12:34:07,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/a812fd6162a940c689e7a5ffc3b35db2 for snapshot=snapshot_backup_system 2023-10-02T12:34:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:07,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:07,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-10-02T12:34:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-10-02T12:34:07,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:07,817 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:07,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-10-02T12:34:07,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 7b374dce01ddb9705e86d3520a03887f in 618 msec 2023-10-02T12:34:07,820 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:07,821 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:07,821 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:07,821 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:34:07,822 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:34:07,836 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:07,856 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:07,857 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:34:07,861 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:07,861 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-10-02T12:34:07,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 680 msec 2023-10-02T12:34:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:34:08,300 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696077240951 ... 2023-10-02T12:34:08,301 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:34:08,302 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:34:08,303 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696077240951 2023-10-02T12:34:08,303 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36711/backupUT/backup_1696077240951/default/test-1696250014034. 2023-10-02T12:34:08,304 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.BackupUtils(465): hdfs://localhost:36711/backupUT/backup_1696077240951/default is empty, remove it. 2023-10-02T12:34:08,323 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:34:08,323 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:34:08,325 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupAdminImpl(276): Delete backup backup_1696077240951 completed. 2023-10-02T12:34:08,338 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1034): test-1696250014034 2023-10-02T12:34:08,344 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:34:08,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:34:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37819 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:34:08,355 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:34:08,355 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:34:08,363 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,363 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] backup.TestBackupDelete(158): Deleted 1 backups. Total older than 1 days: 1 2023-10-02T12:34:08,390 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(175): after: backup.TestBackupDelete#testBackupPurgeOldBackupsCommand Thread=609 (was 602) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:48188 [Waiting for operation #13] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: regionserver/jenkins-hbase3:0.Chore.2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:48108 [Waiting for operation #15] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: regionserver/jenkins-hbase3:0.Chore.3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-20 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1566406829-172.31.12.81-1696249997240:blk_1073741894_1070, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:65053@0x70c4eaec java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1529) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1512) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1481) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$496/1312974157.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:65053@0x70c4eaec-SendThread(127.0.0.1:65053) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: hconnection-0x595731fb-shared-pool-21 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-18 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:48024 [Waiting for operation #11] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:65053@0x70c4eaec-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x595731fb-shared-pool-19 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x6eb07023-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:48168 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1520423817_16 at /127.0.0.1:48006 [Receiving block BP-1566406829-172.31.12.81-1696249997240:blk_1073741895_1071] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x595731fb-shared-pool-17 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1566406829-172.31.12.81-1696249997240:blk_1073741895_1071, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1839389754_16 at /127.0.0.1:47972 [Waiting for operation #20] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1520423817_16 at /127.0.0.1:47996 [Receiving block BP-1566406829-172.31.12.81-1696249997240:blk_1073741894_1070] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=702 (was 683) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=517 (was 512) - SystemLoadAverage LEAK? -, ProcessCount=184 (was 184), AvailableMemoryMB=1185 (was 1229) 2023-10-02T12:34:08,390 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.ResourceChecker(130): Thread=609 is superior to 500 2023-10-02T12:34:08,399 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:34:08,400 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,401 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:34:08,401 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] util.JVMClusterUtil(257): Found active master hash=2089515604, stopped=false 2023-10-02T12:34:08,401 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:34:08,472 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:34:08,474 DEBUG [Listener at jenkins-hbase3.apache.org/34913 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,474 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,40937,1696250003832' ***** 2023-10-02T12:34:08,474 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:34:08,474 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:34:08,492 INFO [RS:0;jenkins-hbase3:40937 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@610d16b9{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:34:08,497 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:34:08,498 INFO [RS:0;jenkins-hbase3:40937 {}] server.AbstractConnector(383): Stopped ServerConnector@36efd4d1{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:08,499 INFO [RS:0;jenkins-hbase3:40937 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:34:08,500 INFO [RS:0;jenkins-hbase3:40937 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1b05b10e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:34:08,501 INFO [RS:0;jenkins-hbase3:40937 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@21a8923c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.log.dir/,STOPPED} 2023-10-02T12:34:08,504 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:34:08,510 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:34:08,510 INFO [RS:0;jenkins-hbase3:40937 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:34:08,511 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:34:08,511 INFO [RS:0;jenkins-hbase3:40937 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:34:08,511 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 9635224d05e38a05440ef6ab163fbe51 2023-10-02T12:34:08,512 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 252a897416193ae7ac42e7e4f2a3268e 2023-10-02T12:34:08,513 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 639d2bc3c5c2cb1c241de32b99c9da77 2023-10-02T12:34:08,513 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 7b374dce01ddb9705e86d3520a03887f 2023-10-02T12:34:08,513 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:08,513 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 0d134e56945ef6d5e83667f94878fc5e 2023-10-02T12:34:08,513 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 3b06f8973540cb8d6705c60d9447cf6c 2023-10-02T12:34:08,513 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,40937,1696250003832 2023-10-02T12:34:08,513 DEBUG [RS:0;jenkins-hbase3:40937 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:08,515 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9635224d05e38a05440ef6ab163fbe51, disabling compactions & flushes 2023-10-02T12:34:08,515 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:34:08,515 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:34:08,515 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. after waiting 0 ms 2023-10-02T12:34:08,515 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:34:08,516 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:34:08,516 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:34:08,516 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:34:08,516 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:34:08,517 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-02T12:34:08,517 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1583): Online Regions={9635224d05e38a05440ef6ab163fbe51=backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51., 252a897416193ae7ac42e7e4f2a3268e=ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e., 639d2bc3c5c2cb1c241de32b99c9da77=ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77., 1588230740=hbase:meta,,1.1588230740, 7b374dce01ddb9705e86d3520a03887f=backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f., edb5468052a27c274e91621fff6dca47=hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47., 0d134e56945ef6d5e83667f94878fc5e=test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e., 3b06f8973540cb8d6705c60d9447cf6c=ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.} 2023-10-02T12:34:08,519 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 1588230740, 252a897416193ae7ac42e7e4f2a3268e, 3b06f8973540cb8d6705c60d9447cf6c, 639d2bc3c5c2cb1c241de32b99c9da77, 7b374dce01ddb9705e86d3520a03887f, 9635224d05e38a05440ef6ab163fbe51, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:08,519 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:34:08,520 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:34:08,520 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:34:08,520 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:34:08,520 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:34:08,520 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=9.25 KB heapSize=16.20 KB 2023-10-02T12:34:08,569 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system_bulk/9635224d05e38a05440ef6ab163fbe51/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:34:08,571 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/.tmp/info/71555634b5c74d518ada0edcf97c64f5 is 159, key is ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77./info:regioninfo/1696250017617/Put/seqid=0 2023-10-02T12:34:08,572 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:08,578 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:34:08,578 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9635224d05e38a05440ef6ab163fbe51: 2023-10-02T12:34:08,578 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696250022190.9635224d05e38a05440ef6ab163fbe51. 2023-10-02T12:34:08,593 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 252a897416193ae7ac42e7e4f2a3268e, disabling compactions & flushes 2023-10-02T12:34:08,593 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:34:08,593 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:34:08,593 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. after waiting 0 ms 2023-10-02T12:34:08,593 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:34:08,614 INFO [master/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(145): Chore: RollingUpgradeChore missed its start time 2023-10-02T12:34:08,631 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns3/test-16962500140342/252a897416193ae7ac42e7e4f2a3268e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:34:08,632 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:08,635 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:34:08,635 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 252a897416193ae7ac42e7e4f2a3268e: 2023-10-02T12:34:08,635 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962500140342,,1696250018017.252a897416193ae7ac42e7e4f2a3268e. 2023-10-02T12:34:08,637 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 639d2bc3c5c2cb1c241de32b99c9da77, disabling compactions & flushes 2023-10-02T12:34:08,637 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:34:08,637 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:34:08,637 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. after waiting 0 ms 2023-10-02T12:34:08,637 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:34:08,637 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 639d2bc3c5c2cb1c241de32b99c9da77 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:34:08,674 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/.tmp/f/cb55bdde61724dca9da9dc085f1e3c12 is 37, key is row10/f:q1/1696250017940/Put/seqid=0 2023-10-02T12:34:08,684 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/.tmp/f/cb55bdde61724dca9da9dc085f1e3c12 2023-10-02T12:34:08,692 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/.tmp/f/cb55bdde61724dca9da9dc085f1e3c12 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/f/cb55bdde61724dca9da9dc085f1e3c12 2023-10-02T12:34:08,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/f/cb55bdde61724dca9da9dc085f1e3c12, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:34:08,707 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 639d2bc3c5c2cb1c241de32b99c9da77 in 70ms, sequenceid=103, compaction requested=false 2023-10-02T12:34:08,719 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 1588230740, 3b06f8973540cb8d6705c60d9447cf6c, 639d2bc3c5c2cb1c241de32b99c9da77, 7b374dce01ddb9705e86d3520a03887f, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:08,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns2/test-16962500140341/639d2bc3c5c2cb1c241de32b99c9da77/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:34:08,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:08,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:34:08,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 639d2bc3c5c2cb1c241de32b99c9da77: 2023-10-02T12:34:08,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962500140341,,1696250016811.639d2bc3c5c2cb1c241de32b99c9da77. 2023-10-02T12:34:08,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7b374dce01ddb9705e86d3520a03887f, disabling compactions & flushes 2023-10-02T12:34:08,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:08,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:08,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. after waiting 0 ms 2023-10-02T12:34:08,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:08,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7b374dce01ddb9705e86d3520a03887f 2/2 column families, dataSize=223 B heapSize=1.01 KB 2023-10-02T12:34:08,734 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/cca74272e25c4f9a85574a97a12c5c31 is 91, key is incrbackupset:hdfs://localhost:36711/backupUT/meta:test-1696250014034/1696250048339/Put/seqid=0 2023-10-02T12:34:08,750 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:34:08,750 INFO [regionserver/jenkins-hbase3:0.Chore.3 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:34:08,920 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 1588230740, 3b06f8973540cb8d6705c60d9447cf6c, 7b374dce01ddb9705e86d3520a03887f, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:08,996 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/.tmp/info/71555634b5c74d518ada0edcf97c64f5 2023-10-02T12:34:09,023 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/.tmp/table/85f20f96342c480da250ab83b0564b84 is 59, key is ns2:test-16962500140341/table:state/1696250017627/Put/seqid=0 2023-10-02T12:34:09,120 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 1588230740, 3b06f8973540cb8d6705c60d9447cf6c, 7b374dce01ddb9705e86d3520a03887f, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:09,147 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/cca74272e25c4f9a85574a97a12c5c31 2023-10-02T12:34:09,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cca74272e25c4f9a85574a97a12c5c31 2023-10-02T12:34:09,158 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/c8aa60c617f74e9b91d0bb9650ad6e5c is 59, key is session:backup_1696077240951/session:/1696250048324/DeleteFamily/seqid=0 2023-10-02T12:34:09,321 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 1588230740, 3b06f8973540cb8d6705c60d9447cf6c, 7b374dce01ddb9705e86d3520a03887f, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:09,436 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=714 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/.tmp/table/85f20f96342c480da250ab83b0564b84 2023-10-02T12:34:09,444 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/.tmp/info/71555634b5c74d518ada0edcf97c64f5 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/info/71555634b5c74d518ada0edcf97c64f5 2023-10-02T12:34:09,450 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/info/71555634b5c74d518ada0edcf97c64f5, entries=70, sequenceid=39, filesize=14.1 K 2023-10-02T12:34:09,452 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/.tmp/table/85f20f96342c480da250ab83b0564b84 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/table/85f20f96342c480da250ab83b0564b84 2023-10-02T12:34:09,462 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/table/85f20f96342c480da250ab83b0564b84, entries=14, sequenceid=39, filesize=5.8 K 2023-10-02T12:34:09,463 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~9.25 KB/9474, heapSize ~15.92 KB/16304, currentSize=0 B/0 for 1588230740 in 943ms, sequenceid=39, compaction requested=false 2023-10-02T12:34:09,474 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/meta/1588230740/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=1 2023-10-02T12:34:09,475 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:09,475 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:34:09,476 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:09,476 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:34:09,476 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:09,521 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-10-02T12:34:09,521 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1583): Online Regions={7b374dce01ddb9705e86d3520a03887f=backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f., edb5468052a27c274e91621fff6dca47=hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47., 0d134e56945ef6d5e83667f94878fc5e=test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e., 3b06f8973540cb8d6705c60d9447cf6c=ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c.} 2023-10-02T12:34:09,521 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 3b06f8973540cb8d6705c60d9447cf6c, 7b374dce01ddb9705e86d3520a03887f, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:09,568 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/c8aa60c617f74e9b91d0bb9650ad6e5c 2023-10-02T12:34:09,577 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8aa60c617f74e9b91d0bb9650ad6e5c 2023-10-02T12:34:09,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/meta/cca74272e25c4f9a85574a97a12c5c31 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/cca74272e25c4f9a85574a97a12c5c31 2023-10-02T12:34:09,599 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cca74272e25c4f9a85574a97a12c5c31 2023-10-02T12:34:09,600 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/meta/cca74272e25c4f9a85574a97a12c5c31, entries=2, sequenceid=72, filesize=5.2 K 2023-10-02T12:34:09,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/.tmp/session/c8aa60c617f74e9b91d0bb9650ad6e5c as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/c8aa60c617f74e9b91d0bb9650ad6e5c 2023-10-02T12:34:09,643 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8aa60c617f74e9b91d0bb9650ad6e5c 2023-10-02T12:34:09,643 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/session/c8aa60c617f74e9b91d0bb9650ad6e5c, entries=2, sequenceid=72, filesize=5.2 K 2023-10-02T12:34:09,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~223 B/223, heapSize ~1000 B/1000, currentSize=0 B/0 for 7b374dce01ddb9705e86d3520a03887f in 918ms, sequenceid=72, compaction requested=true 2023-10-02T12:34:09,691 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/backup/system/7b374dce01ddb9705e86d3520a03887f/recovered.edits/75.seqid, newMaxSeqId=75, maxSeqId=1 2023-10-02T12:34:09,692 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:09,693 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:09,693 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7b374dce01ddb9705e86d3520a03887f: 2023-10-02T12:34:09,693 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696250021559.7b374dce01ddb9705e86d3520a03887f. 2023-10-02T12:34:09,694 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing edb5468052a27c274e91621fff6dca47, disabling compactions & flushes 2023-10-02T12:34:09,694 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:34:09,694 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:34:09,694 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. after waiting 0 ms 2023-10-02T12:34:09,694 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:34:09,695 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing edb5468052a27c274e91621fff6dca47 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:34:09,715 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/.tmp/info/9c5bb8cc6baa4d0ca6af80f5a96ebd09 is 45, key is default/info:d/1696250008430/Put/seqid=0 2023-10-02T12:34:09,721 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 3b06f8973540cb8d6705c60d9447cf6c, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:09,922 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 3b06f8973540cb8d6705c60d9447cf6c, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:10,122 DEBUG [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1609): Waiting on 0d134e56945ef6d5e83667f94878fc5e, 3b06f8973540cb8d6705c60d9447cf6c, edb5468052a27c274e91621fff6dca47 2023-10-02T12:34:10,123 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/.tmp/info/9c5bb8cc6baa4d0ca6af80f5a96ebd09 2023-10-02T12:34:10,132 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/.tmp/info/9c5bb8cc6baa4d0ca6af80f5a96ebd09 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/info/9c5bb8cc6baa4d0ca6af80f5a96ebd09 2023-10-02T12:34:10,138 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/info/9c5bb8cc6baa4d0ca6af80f5a96ebd09, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:34:10,139 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for edb5468052a27c274e91621fff6dca47 in 445ms, sequenceid=11, compaction requested=false 2023-10-02T12:34:10,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/hbase/namespace/edb5468052a27c274e91621fff6dca47/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:34:10,147 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:10,148 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:34:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for edb5468052a27c274e91621fff6dca47: 2023-10-02T12:34:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696250007324.edb5468052a27c274e91621fff6dca47. 2023-10-02T12:34:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0d134e56945ef6d5e83667f94878fc5e, disabling compactions & flushes 2023-10-02T12:34:10,148 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. after waiting 0 ms 2023-10-02T12:34:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:10,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/default/test-1696250014034/0d134e56945ef6d5e83667f94878fc5e/recovered.edits/108.seqid, newMaxSeqId=108, maxSeqId=1 2023-10-02T12:34:10,178 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:10,179 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:10,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0d134e56945ef6d5e83667f94878fc5e: 2023-10-02T12:34:10,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696250014034,,1696250014536.0d134e56945ef6d5e83667f94878fc5e. 2023-10-02T12:34:10,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3b06f8973540cb8d6705c60d9447cf6c, disabling compactions & flushes 2023-10-02T12:34:10,181 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:34:10,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:34:10,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. after waiting 0 ms 2023-10-02T12:34:10,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:34:10,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/data/ns4/test-16962500140343/3b06f8973540cb8d6705c60d9447cf6c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:34:10,186 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:10,186 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:34:10,187 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3b06f8973540cb8d6705c60d9447cf6c: 2023-10-02T12:34:10,187 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962500140343,,1696250020144.3b06f8973540cb8d6705c60d9447cf6c. 2023-10-02T12:34:10,322 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,40937,1696250003832; all regions closed. 2023-10-02T12:34:10,333 DEBUG [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(1062): Moved 2 WAL file(s) to /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs 2023-10-02T12:34:10,333 INFO [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832.meta:.meta(num 1696250034492) 2023-10-02T12:34:10,352 DEBUG [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/oldWALs 2023-10-02T12:34:10,352 INFO [RS:0;jenkins-hbase3:40937 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40937%2C1696250003832:(num 1696250034464) 2023-10-02T12:34:10,352 DEBUG [RS:0;jenkins-hbase3:40937 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:10,353 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:34:10,354 INFO [RS:0;jenkins-hbase3:40937 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-10-02T12:34:10,354 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:34:10,357 INFO [RS:0;jenkins-hbase3:40937 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:40937 2023-10-02T12:34:10,370 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,40937,1696250003832] 2023-10-02T12:34:10,370 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,40937,1696250003832; numProcessing=1 2023-10-02T12:34:10,372 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,40937,1696250003832 expired; onlineServers=0 2023-10-02T12:34:10,372 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37819,1696250000621' ***** 2023-10-02T12:34:10,372 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:34:10,374 DEBUG [M:0;jenkins-hbase3:37819 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be3e558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:34:10,374 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:34:10,438 INFO [M:0;jenkins-hbase3:37819 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@210ce6e1{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:34:10,439 INFO [M:0;jenkins-hbase3:37819 {}] server.AbstractConnector(383): Stopped ServerConnector@761be04a{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:10,439 INFO [M:0;jenkins-hbase3:37819 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:34:10,440 INFO [M:0;jenkins-hbase3:37819 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5fe61d87{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:34:10,441 INFO [M:0;jenkins-hbase3:37819 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4ebc82c4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/hadoop.log.dir/,STOPPED} 2023-10-02T12:34:10,441 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37819,1696250000621 2023-10-02T12:34:10,442 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37819,1696250000621; all regions closed. 2023-10-02T12:34:10,442 DEBUG [M:0;jenkins-hbase3:37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:10,442 INFO [M:0;jenkins-hbase3:37819 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:34:10,442 INFO [M:0;jenkins-hbase3:37819 {}] server.AbstractConnector(383): Stopped ServerConnector@5a1ea680{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:34:10,443 DEBUG [M:0;jenkins-hbase3:37819 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:34:10,443 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:34:10,443 INFO [M:0;jenkins-hbase3:37819 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:34:10,444 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250005554 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250005554,5,FailOnTimeoutGroup] 2023-10-02T12:34:10,444 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250005554 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250005554,5,FailOnTimeoutGroup] 2023-10-02T12:34:10,444 DEBUG [M:0;jenkins-hbase3:37819 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:10,444 DEBUG [M:0;jenkins-hbase3:37819 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:34:10,444 INFO [M:0;jenkins-hbase3:37819 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:34:10,444 DEBUG [M:0;jenkins-hbase3:37819 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:34:10,444 INFO [M:0;jenkins-hbase3:37819 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:34:10,444 ERROR [M:0;jenkins-hbase3:37819 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT 2023-10-02T12:34:10,447 INFO [M:0;jenkins-hbase3:37819 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:34:10,447 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:34:10,483 INFO [RS:0;jenkins-hbase3:40937 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,40937,1696250003832; zookeeper connection closed. 2023-10-02T12:34:10,484 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fe82b68 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fe82b68 2023-10-02T12:34:10,484 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:34:10,486 WARN [M:0;jenkins-hbase3:37819 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:34:10,486 INFO [M:0;jenkins-hbase3:37819 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:34:10,486 INFO [M:0;jenkins-hbase3:37819 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:34:10,488 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:34:10,505 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:10,505 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:10,505 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:34:10,505 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:10,506 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=195.20 KB heapSize=236.14 KB 2023-10-02T12:34:10,521 DEBUG [M:0;jenkins-hbase3:37819 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47de35271a8a46a3908806cbe46b3099 is 88, key is hbase:meta,,1/info:sn/1696250006578/Put/seqid=0 2023-10-02T12:34:10,929 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47de35271a8a46a3908806cbe46b3099 2023-10-02T12:34:10,963 DEBUG [M:0;jenkins-hbase3:37819 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d092afb13b9e49618dd9145649f2491f is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1696250022609/Put/seqid=0 2023-10-02T12:34:10,974 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=194.61 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d092afb13b9e49618dd9145649f2491f 2023-10-02T12:34:10,981 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d092afb13b9e49618dd9145649f2491f 2023-10-02T12:34:10,999 DEBUG [M:0;jenkins-hbase3:37819 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ac7d9ae5defc4337bdc988acaabfdb08 is 82, key is jenkins-hbase3.apache.org,40937,1696250003832/rs:state/1696250005611/Put/seqid=0 2023-10-02T12:34:11,406 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ac7d9ae5defc4337bdc988acaabfdb08 2023-10-02T12:34:11,412 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47de35271a8a46a3908806cbe46b3099 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/47de35271a8a46a3908806cbe46b3099 2023-10-02T12:34:11,419 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/47de35271a8a46a3908806cbe46b3099, entries=8, sequenceid=483, filesize=5.5 K 2023-10-02T12:34:11,420 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d092afb13b9e49618dd9145649f2491f as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d092afb13b9e49618dd9145649f2491f 2023-10-02T12:34:11,426 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d092afb13b9e49618dd9145649f2491f 2023-10-02T12:34:11,426 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d092afb13b9e49618dd9145649f2491f, entries=51, sequenceid=483, filesize=16.1 K 2023-10-02T12:34:11,430 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ac7d9ae5defc4337bdc988acaabfdb08 as hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ac7d9ae5defc4337bdc988acaabfdb08 2023-10-02T12:34:11,436 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36711/user/jenkins/test-data/67abf3bd-fa20-395e-f798-961ec4774664/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ac7d9ae5defc4337bdc988acaabfdb08, entries=1, sequenceid=483, filesize=5.1 K 2023-10-02T12:34:11,437 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(3022): Finished flush of dataSize ~195.20 KB/199885, heapSize ~235.84 KB/241504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 932ms, sequenceid=483, compaction requested=false 2023-10-02T12:34:11,443 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:11,443 DEBUG [M:0;jenkins-hbase3:37819 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:34:11,447 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:34:11,447 INFO [M:0;jenkins-hbase3:37819 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:34:11,447 INFO [M:0;jenkins-hbase3:37819 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:34:11,448 INFO [M:0;jenkins-hbase3:37819 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37819 2023-10-02T12:34:11,553 INFO [M:0;jenkins-hbase3:37819 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37819,1696250000621; zookeeper connection closed. 2023-10-02T12:34:11,556 WARN [Listener at jenkins-hbase3.apache.org/34913 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:34:11,563 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:11,677 WARN [BP-1566406829-172.31.12.81-1696249997240 heartbeating to localhost/127.0.0.1:36711 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:34:11,677 WARN [BP-1566406829-172.31.12.81-1696249997240 heartbeating to localhost/127.0.0.1:36711 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1566406829-172.31.12.81-1696249997240 (Datanode Uuid 1cc1d88c-4323-4d6a-95ba-5ffdfafaf159) service to localhost/127.0.0.1:36711 2023-10-02T12:34:11,679 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/dfs/data/data1/current/BP-1566406829-172.31.12.81-1696249997240 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,679 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97f5c5b2-8970-a7af-862a-16dc13162845/cluster_4d2cf06e-a08c-d137-cf3d-a5fe6796f830/dfs/data/data2/current/BP-1566406829-172.31.12.81-1696249997240 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:34:11,724 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:34:11,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:34:11,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962500140341 2023-10-02T12:34:11,792 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:34:11,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:34:11,794 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:34:11,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:34:11,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:34:11,796 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962500140343 2023-10-02T12:34:11,797 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962500140342 2023-10-02T12:34:11,798 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696250014034 2023-10-02T12:34:11,798 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:11,883 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:34:11,883 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:34:11,894 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:12,016 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:12,130 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:34:12,131 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:12,238 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:34:12,245 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:34:12,247 ERROR [Thread[Thread-286,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:34:12,254 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:34:12,357 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:34:12,357 INFO [Listener at jenkins-hbase3.apache.org/34913 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped