2024-11-22 16:43:51,396 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 2024-11-22 16:43:51,407 main DEBUG Took 0.009309 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 16:43:51,407 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 16:43:51,407 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 16:43:51,408 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 16:43:51,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,416 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 16:43:51,431 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,434 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,435 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,436 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,436 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,437 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,438 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,439 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,440 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,440 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,441 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,442 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:43:51,442 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,443 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 16:43:51,444 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:43:51,445 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 16:43:51,447 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 16:43:51,447 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 16:43:51,449 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 16:43:51,449 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 16:43:51,458 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 16:43:51,460 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 16:43:51,462 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 16:43:51,462 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 16:43:51,462 main DEBUG createAppenders(={Console}) 2024-11-22 16:43:51,463 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 initialized 2024-11-22 16:43:51,464 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 2024-11-22 16:43:51,464 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 OK. 2024-11-22 16:43:51,464 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 16:43:51,465 main DEBUG OutputStream closed 2024-11-22 16:43:51,465 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 16:43:51,465 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 16:43:51,465 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4803b726 OK 2024-11-22 16:43:51,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 16:43:51,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 16:43:51,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 16:43:51,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 16:43:51,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 16:43:51,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 16:43:51,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 16:43:51,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 16:43:51,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 16:43:51,538 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 16:43:51,538 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 16:43:51,538 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 16:43:51,538 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 16:43:51,539 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 16:43:51,539 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 16:43:51,539 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 16:43:51,540 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 16:43:51,540 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 16:43:51,543 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 16:43:51,543 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null 2024-11-22 16:43:51,543 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 16:43:51,544 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK. 2024-11-22T16:43:51,814 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5 2024-11-22 16:43:51,817 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 16:43:51,817 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T16:43:51,827 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure timeout: 13 mins 2024-11-22T16:43:51,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T16:43:51,864 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3, deleteOnExit=true 2024-11-22T16:43:51,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T16:43:51,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/test.cache.data in system properties and HBase conf 2024-11-22T16:43:51,866 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T16:43:51,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/hadoop.log.dir in system properties and HBase conf 2024-11-22T16:43:51,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T16:43:51,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T16:43:51,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T16:43:51,959 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T16:43:52,054 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:43:52,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T16:43:52,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T16:43:52,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T16:43:52,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T16:43:52,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T16:43:52,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T16:43:52,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T16:43:52,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T16:43:52,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T16:43:52,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/nfs.dump.dir in system properties and HBase conf 2024-11-22T16:43:52,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/java.io.tmpdir in system properties and HBase conf 2024-11-22T16:43:52,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T16:43:52,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T16:43:52,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T16:43:52,897 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T16:43:52,981 INFO [Time-limited test {}] log.Log(170): Logging initialized @2296ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T16:43:53,065 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T16:43:53,127 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T16:43:53,148 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T16:43:53,149 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T16:43:53,150 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T16:43:53,164 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T16:43:53,167 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/hadoop.log.dir/,AVAILABLE} 2024-11-22T16:43:53,168 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T16:43:53,367 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27b19a13{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/java.io.tmpdir/jetty-localhost-45473-hadoop-hdfs-3_4_1-tests_jar-_-any-2986063021326478359/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T16:43:53,375 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:45473} 2024-11-22T16:43:53,375 INFO [Time-limited test {}] server.Server(415): Started @2690ms 2024-11-22T16:43:53,859 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T16:43:53,867 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T16:43:53,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T16:43:53,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T16:43:53,868 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T16:43:53,869 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/hadoop.log.dir/,AVAILABLE} 2024-11-22T16:43:53,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T16:43:54,000 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@156ffeaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/java.io.tmpdir/jetty-localhost-34751-hadoop-hdfs-3_4_1-tests_jar-_-any-16569683479150761227/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T16:43:54,002 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:34751} 2024-11-22T16:43:54,002 INFO [Time-limited test {}] server.Server(415): Started @3317ms 2024-11-22T16:43:54,068 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T16:43:54,576 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/data/data2/current/BP-887821746-172.17.0.2-1732293832654/current, will proceed with Du for space computation calculation, 2024-11-22T16:43:54,576 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/data/data1/current/BP-887821746-172.17.0.2-1732293832654/current, will proceed with Du for space computation calculation, 2024-11-22T16:43:54,632 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T16:43:54,714 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcdc48c5975cbe6de with lease ID 0xce9b5a46998d91eb: Processing first storage report for DS-2143ad93-a61b-44f0-9ca7-aca932bdc1bd from datanode DatanodeRegistration(127.0.0.1:45621, datanodeUuid=d2b690e1-a7e7-43f3-b22b-ba19fa955d8c, infoPort=42723, infoSecurePort=0, ipcPort=46293, storageInfo=lv=-57;cid=testClusterID;nsid=1226056609;c=1732293832654) 2024-11-22T16:43:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcdc48c5975cbe6de with lease ID 0xce9b5a46998d91eb: from storage DS-2143ad93-a61b-44f0-9ca7-aca932bdc1bd node DatanodeRegistration(127.0.0.1:45621, datanodeUuid=d2b690e1-a7e7-43f3-b22b-ba19fa955d8c, infoPort=42723, infoSecurePort=0, ipcPort=46293, storageInfo=lv=-57;cid=testClusterID;nsid=1226056609;c=1732293832654), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-22T16:43:54,716 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5 2024-11-22T16:43:54,717 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcdc48c5975cbe6de with lease ID 0xce9b5a46998d91eb: Processing first storage report for DS-4055a426-2d59-41c6-82d5-a67607f67d06 from datanode DatanodeRegistration(127.0.0.1:45621, datanodeUuid=d2b690e1-a7e7-43f3-b22b-ba19fa955d8c, infoPort=42723, infoSecurePort=0, ipcPort=46293, storageInfo=lv=-57;cid=testClusterID;nsid=1226056609;c=1732293832654) 2024-11-22T16:43:54,717 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcdc48c5975cbe6de with lease ID 0xce9b5a46998d91eb: from storage DS-4055a426-2d59-41c6-82d5-a67607f67d06 node DatanodeRegistration(127.0.0.1:45621, datanodeUuid=d2b690e1-a7e7-43f3-b22b-ba19fa955d8c, infoPort=42723, infoSecurePort=0, ipcPort=46293, storageInfo=lv=-57;cid=testClusterID;nsid=1226056609;c=1732293832654), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T16:43:54,807 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/zookeeper_0, clientPort=59446, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T16:43:54,818 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59446 2024-11-22T16:43:54,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:54,838 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:55,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741825_1001 (size=7) 2024-11-22T16:43:55,512 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 with version=8 2024-11-22T16:43:55,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/hbase-staging 2024-11-22T16:43:55,617 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T16:43:55,921 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:43:55,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:43:55,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:43:55,940 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:43:55,940 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:43:55,940 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:43:56,106 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T16:43:56,198 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T16:43:56,211 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T16:43:56,217 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:43:56,258 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9304 (auto-detected) 2024-11-22T16:43:56,259 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T16:43:56,286 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45935 2024-11-22T16:43:56,310 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45935 connecting to ZooKeeper ensemble=127.0.0.1:59446 2024-11-22T16:43:56,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459350x0, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:43:56,359 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45935-0x101409022b50000 connected 2024-11-22T16:43:56,403 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:56,406 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:56,422 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:43:56,427 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1, hbase.cluster.distributed=false 2024-11-22T16:43:56,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:43:56,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45935 2024-11-22T16:43:56,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45935 2024-11-22T16:43:56,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45935 2024-11-22T16:43:56,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45935 2024-11-22T16:43:56,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45935 2024-11-22T16:43:56,634 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=45 2024-11-22T16:43:56,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:43:56,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:43:56,638 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:43:56,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:43:56,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:43:56,642 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:43:56,646 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:43:56,647 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46439 2024-11-22T16:43:56,650 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46439 connecting to ZooKeeper ensemble=127.0.0.1:59446 2024-11-22T16:43:56,652 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:56,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:56,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464390x0, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:43:56,668 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464390x0, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:43:56,673 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:43:56,674 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46439-0x101409022b50001 connected 2024-11-22T16:43:56,684 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:43:56,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T16:43:56,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:43:56,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46439 2024-11-22T16:43:56,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46439 2024-11-22T16:43:56,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46439 2024-11-22T16:43:56,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46439 2024-11-22T16:43:56,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46439 2024-11-22T16:43:56,717 DEBUG [M:0;4618c6346924:45935 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4618c6346924:45935 2024-11-22T16:43:56,718 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4618c6346924,45935,1732293835695 2024-11-22T16:43:56,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:43:56,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:43:56,729 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4618c6346924,45935,1732293835695 2024-11-22T16:43:56,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T16:43:56,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:56,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:56,752 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T16:43:56,753 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4618c6346924,45935,1732293835695 from backup master directory 2024-11-22T16:43:56,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:43:56,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4618c6346924,45935,1732293835695 2024-11-22T16:43:56,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:43:56,759 WARN [master/4618c6346924:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:43:56,759 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4618c6346924,45935,1732293835695 2024-11-22T16:43:56,762 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T16:43:56,764 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T16:43:56,824 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/hbase.id] with ID: f073a1fb-7d3e-4875-a75a-b7d7fc45c4d9 2024-11-22T16:43:56,824 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/.tmp/hbase.id 2024-11-22T16:43:56,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741826_1002 (size=42) 2024-11-22T16:43:56,837 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/.tmp/hbase.id]:[hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/hbase.id] 2024-11-22T16:43:56,888 INFO [master/4618c6346924:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:43:56,893 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T16:43:56,913 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-22T16:43:56,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:56,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:56,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741827_1003 (size=196) 2024-11-22T16:43:56,956 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:43:56,959 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T16:43:56,986 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:43:56,992 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:43:57,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741828_1004 (size=1189) 2024-11-22T16:43:57,051 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store 2024-11-22T16:43:57,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741829_1005 (size=34) 2024-11-22T16:43:57,476 INFO [master/4618c6346924:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T16:43:57,479 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:43:57,481 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:43:57,481 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:43:57,481 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:43:57,483 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:43:57,483 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:43:57,483 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:43:57,485 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732293837481Disabling compacts and flushes for region at 1732293837481Disabling writes for close at 1732293837483 (+2 ms)Writing region close event to WAL at 1732293837483Closed at 1732293837483 2024-11-22T16:43:57,487 WARN [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/.initializing 2024-11-22T16:43:57,487 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/WALs/4618c6346924,45935,1732293835695 2024-11-22T16:43:57,498 INFO [master/4618c6346924:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:43:57,517 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C45935%2C1732293835695, suffix=, logDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/WALs/4618c6346924,45935,1732293835695, archiveDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/oldWALs, maxLogs=10 2024-11-22T16:43:57,555 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/WALs/4618c6346924,45935,1732293835695/4618c6346924%2C45935%2C1732293835695.1732293837524, exclude list is [], retry=0 2024-11-22T16:43:57,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45621,DS-2143ad93-a61b-44f0-9ca7-aca932bdc1bd,DISK] 2024-11-22T16:43:57,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T16:43:57,621 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/WALs/4618c6346924,45935,1732293835695/4618c6346924%2C45935%2C1732293835695.1732293837524 2024-11-22T16:43:57,623 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-22T16:43:57,624 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:43:57,624 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:43:57,628 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,629 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,703 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T16:43:57,707 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:57,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:57,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T16:43:57,714 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:57,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:43:57,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T16:43:57,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:57,724 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:43:57,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T16:43:57,729 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:57,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:43:57,733 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,737 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,739 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,745 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,746 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,751 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:43:57,755 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:43:57,761 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:43:57,762 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66019002, jitterRate=-0.016240209341049194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:43:57,768 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732293837644Initializing all the Stores at 1732293837647 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293837648 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293837649 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293837651 (+2 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293837651Cleaning up temporary data from old regions at 1732293837746 (+95 ms)Region opened successfully at 1732293837768 (+22 ms) 2024-11-22T16:43:57,771 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T16:43:57,808 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753bbbdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:43:57,843 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T16:43:57,855 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:43:57,855 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:43:57,859 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:43:57,860 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:43:57,865 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:43:57,866 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T16:43:57,898 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T16:43:57,910 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T16:43:57,912 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T16:43:57,915 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T16:43:57,917 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T16:43:57,918 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T16:43:57,921 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T16:43:57,925 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T16:43:57,926 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T16:43:57,928 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T16:43:57,932 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T16:43:57,952 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T16:43:57,953 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T16:43:57,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:43:57,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:43:57,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:57,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:57,961 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4618c6346924,45935,1732293835695, sessionid=0x101409022b50000, setting cluster-up flag (Was=false) 2024-11-22T16:43:57,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:57,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:57,981 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T16:43:57,983 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,45935,1732293835695 2024-11-22T16:43:57,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:57,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:57,999 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T16:43:58,001 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,45935,1732293835695 2024-11-22T16:43:58,008 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T16:43:58,097 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:43:58,105 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(746): ClusterId : f073a1fb-7d3e-4875-a75a-b7d7fc45c4d9 2024-11-22T16:43:58,108 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:43:58,109 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T16:43:58,114 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:43:58,114 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:43:58,117 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T16:43:58,118 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:43:58,119 DEBUG [RS:0;4618c6346924:46439 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6362b74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:43:58,125 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4618c6346924,45935,1732293835695 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T16:43:58,135 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:43:58,135 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:43:58,135 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:43:58,136 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:43:58,136 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4618c6346924:0, corePoolSize=10, maxPoolSize=10 2024-11-22T16:43:58,136 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,136 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:43:58,137 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,140 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4618c6346924:46439 2024-11-22T16:43:58,141 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293868141 2024-11-22T16:43:58,143 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T16:43:58,144 INFO [RS:0;4618c6346924:46439 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:43:58,145 INFO [RS:0;4618c6346924:46439 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:43:58,145 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T16:43:58,145 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:43:58,148 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:43:58,149 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T16:43:58,149 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T16:43:58,149 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,45935,1732293835695 with port=46439, startcode=1732293836577 2024-11-22T16:43:58,150 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T16:43:58,150 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T16:43:58,150 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T16:43:58,151 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,155 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T16:43:58,156 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T16:43:58,157 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T16:43:58,158 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:58,158 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T16:43:58,163 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T16:43:58,163 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T16:43:58,165 DEBUG [RS:0;4618c6346924:46439 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:43:58,168 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293838164,5,FailOnTimeoutGroup] 2024-11-22T16:43:58,176 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293838169,5,FailOnTimeoutGroup] 2024-11-22T16:43:58,176 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,177 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T16:43:58,178 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,179 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741831_1007 (size=1321) 2024-11-22T16:43:58,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43299, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:43:58,216 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,46439,1732293836577 2024-11-22T16:43:58,220 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] master.ServerManager(517): Registering regionserver=4618c6346924,46439,1732293836577 2024-11-22T16:43:58,239 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:43:58,239 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43711 2024-11-22T16:43:58,239 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:43:58,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:43:58,245 DEBUG [RS:0;4618c6346924:46439 {}] zookeeper.ZKUtil(111): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,46439,1732293836577 2024-11-22T16:43:58,245 WARN [RS:0;4618c6346924:46439 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:43:58,245 INFO [RS:0;4618c6346924:46439 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:43:58,245 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577 2024-11-22T16:43:58,247 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,46439,1732293836577] 2024-11-22T16:43:58,280 INFO [RS:0;4618c6346924:46439 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:43:58,299 INFO [RS:0;4618c6346924:46439 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:43:58,306 INFO [RS:0;4618c6346924:46439 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:43:58,306 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,307 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:43:58,315 INFO [RS:0;4618c6346924:46439 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:43:58,317 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,317 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,317 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,317 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,318 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,318 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,318 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:43:58,318 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,318 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,319 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,319 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,319 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,319 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:43:58,320 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:43:58,320 DEBUG [RS:0;4618c6346924:46439 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:43:58,324 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,325 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,325 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,325 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,325 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,325 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,46439,1732293836577-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:43:58,348 INFO [RS:0;4618c6346924:46439 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:43:58,350 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,46439,1732293836577-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,351 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,351 INFO [RS:0;4618c6346924:46439 {}] regionserver.Replication(171): 4618c6346924,46439,1732293836577 started 2024-11-22T16:43:58,370 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:58,371 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,46439,1732293836577, RpcServer on 4618c6346924/172.17.0.2:46439, sessionid=0x101409022b50001 2024-11-22T16:43:58,372 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:43:58,372 DEBUG [RS:0;4618c6346924:46439 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,46439,1732293836577 2024-11-22T16:43:58,372 DEBUG [RS:0;4618c6346924:46439 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,46439,1732293836577' 2024-11-22T16:43:58,372 DEBUG [RS:0;4618c6346924:46439 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:43:58,373 DEBUG [RS:0;4618c6346924:46439 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:43:58,374 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:43:58,374 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:43:58,374 DEBUG [RS:0;4618c6346924:46439 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,46439,1732293836577 2024-11-22T16:43:58,374 DEBUG [RS:0;4618c6346924:46439 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,46439,1732293836577' 2024-11-22T16:43:58,374 DEBUG [RS:0;4618c6346924:46439 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:43:58,375 DEBUG [RS:0;4618c6346924:46439 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:43:58,376 DEBUG [RS:0;4618c6346924:46439 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:43:58,376 INFO [RS:0;4618c6346924:46439 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:43:58,376 INFO [RS:0;4618c6346924:46439 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:43:58,482 INFO [RS:0;4618c6346924:46439 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:43:58,485 INFO [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C46439%2C1732293836577, suffix=, logDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577, archiveDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/oldWALs, maxLogs=32 2024-11-22T16:43:58,503 DEBUG [RS:0;4618c6346924:46439 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577/4618c6346924%2C46439%2C1732293836577.1732293838488, exclude list is [], retry=0 2024-11-22T16:43:58,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45621,DS-2143ad93-a61b-44f0-9ca7-aca932bdc1bd,DISK] 2024-11-22T16:43:58,512 INFO [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577/4618c6346924%2C46439%2C1732293836577.1732293838488 2024-11-22T16:43:58,512 DEBUG [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-22T16:43:58,582 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T16:43:58,583 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:43:58,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741833_1009 (size=32) 2024-11-22T16:43:58,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:43:59,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T16:43:59,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:43:59,010 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T16:43:59,020 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:43:59,020 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T16:43:59,027 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:43:59,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T16:43:59,034 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:43:59,035 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:43:59,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740 2024-11-22T16:43:59,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740 2024-11-22T16:43:59,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:43:59,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:43:59,047 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:43:59,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:43:59,060 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:43:59,062 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65799932, jitterRate=-0.0195046067237854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:43:59,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732293838997Initializing all the Stores at 1732293839000 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293839000Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293839000Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293839000Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293839000Cleaning up temporary data from old regions at 1732293839046 (+46 ms)Region opened successfully at 1732293839065 (+19 ms) 2024-11-22T16:43:59,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:43:59,066 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:43:59,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:43:59,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:43:59,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:43:59,068 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:43:59,068 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732293839066Disabling compacts and flushes for region at 1732293839066Disabling writes for close at 1732293839066Writing region close event to WAL at 1732293839068 (+2 ms)Closed at 1732293839068 2024-11-22T16:43:59,074 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:43:59,074 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T16:43:59,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T16:43:59,105 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:43:59,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T16:43:59,262 DEBUG [4618c6346924:45935 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T16:43:59,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:43:59,287 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPENING 2024-11-22T16:43:59,292 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T16:43:59,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:59,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:43:59,295 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:43:59,295 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:43:59,297 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:43:59,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:43:59,479 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T16:43:59,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T16:43:59,499 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T16:43:59,499 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:43:59,500 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T16:43:59,504 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C46439%2C1732293836577.meta, suffix=.meta, logDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577, archiveDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/oldWALs, maxLogs=32 2024-11-22T16:43:59,527 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577/4618c6346924%2C46439%2C1732293836577.meta.1732293839507.meta, exclude list is [], retry=0 2024-11-22T16:43:59,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45621,DS-2143ad93-a61b-44f0-9ca7-aca932bdc1bd,DISK] 2024-11-22T16:43:59,537 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/WALs/4618c6346924,46439,1732293836577/4618c6346924%2C46439%2C1732293836577.meta.1732293839507.meta 2024-11-22T16:43:59,538 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42723:42723)] 2024-11-22T16:43:59,538 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:43:59,541 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T16:43:59,545 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T16:43:59,554 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T16:43:59,559 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T16:43:59,560 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:43:59,560 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T16:43:59,561 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T16:43:59,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T16:43:59,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:43:59,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T16:43:59,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:43:59,574 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T16:43:59,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:43:59,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T16:43:59,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:43:59,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:43:59,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:43:59,585 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:43:59,589 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740 2024-11-22T16:43:59,592 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740 2024-11-22T16:43:59,595 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:43:59,596 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:43:59,597 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:43:59,600 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:43:59,603 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63851266, jitterRate=-0.04854199290275574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:43:59,603 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T16:43:59,605 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732293839561Writing region info on filesystem at 1732293839561Initializing all the Stores at 1732293839567 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293839568 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293839568Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293839568Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732293839568Cleaning up temporary data from old regions at 1732293839596 (+28 ms)Running coprocessor post-open hooks at 1732293839603 (+7 ms)Region opened successfully at 1732293839605 (+2 ms) 2024-11-22T16:43:59,615 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732293839469 2024-11-22T16:43:59,632 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T16:43:59,632 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T16:43:59,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:43:59,638 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:43:59,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:43:59,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:43:59,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:43:59,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:43:59,646 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4618c6346924,46439,1732293836577 2024-11-22T16:43:59,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T16:43:59,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,46439,1732293836577 in 347 msec 2024-11-22T16:43:59,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T16:43:59,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 572 msec 2024-11-22T16:43:59,666 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:43:59,666 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T16:43:59,691 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:43:59,693 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,46439,1732293836577, seqNum=-1] 2024-11-22T16:43:59,723 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:43:59,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33707, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:43:59,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7110 sec 2024-11-22T16:43:59,756 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732293839756, completionTime=-1 2024-11-22T16:43:59,760 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T16:43:59,760 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:43:59,798 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:43:59,798 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293899798 2024-11-22T16:43:59,798 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293959798 2024-11-22T16:43:59,798 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 37 msec 2024-11-22T16:43:59,801 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,45935,1732293835695-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,802 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,45935,1732293835695-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,802 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,45935,1732293835695-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,804 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4618c6346924:45935, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,804 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,805 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,813 DEBUG [master/4618c6346924:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T16:43:59,840 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.080sec 2024-11-22T16:43:59,841 INFO [master/4618c6346924:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T16:43:59,843 INFO [master/4618c6346924:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T16:43:59,844 INFO [master/4618c6346924:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T16:43:59,844 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T16:43:59,845 INFO [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T16:43:59,845 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,45935,1732293835695-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:43:59,846 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,45935,1732293835695-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T16:43:59,855 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T16:43:59,857 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T16:43:59,857 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,45935,1732293835695-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:43:59,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d7c978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:43:59,924 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T16:43:59,924 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T16:43:59,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4618c6346924,45935,-1 for getting cluster id 2024-11-22T16:43:59,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T16:43:59,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f073a1fb-7d3e-4875-a75a-b7d7fc45c4d9' 2024-11-22T16:43:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T16:43:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f073a1fb-7d3e-4875-a75a-b7d7fc45c4d9" 2024-11-22T16:43:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af2be17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:43:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4618c6346924,45935,-1] 2024-11-22T16:43:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T16:43:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:43:59,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T16:43:59,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df26bdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:43:59,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:43:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,46439,1732293836577, seqNum=-1] 2024-11-22T16:43:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:43:59,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:43:59,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4618c6346924,45935,1732293835695 2024-11-22T16:44:00,003 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTable Thread=207, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=8314 2024-11-22T16:44:00,007 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T16:44:00,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:00,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:44:00,025 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:00,028 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:44:00,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741835_1011 (size=639) 2024-11-22T16:44:00,455 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 827a188ec32f56dcad90ad604a5da931, NAME => 'testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:44:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741836_1012 (size=51) 2024-11-22T16:44:00,882 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:00,882 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1722): Closing 827a188ec32f56dcad90ad604a5da931, disabling compactions & flushes 2024-11-22T16:44:00,882 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:00,882 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:00,882 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. after waiting 0 ms 2024-11-22T16:44:00,882 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:00,882 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:00,882 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 827a188ec32f56dcad90ad604a5da931: Waiting for close lock at 1732293840882Disabling compacts and flushes for region at 1732293840882Disabling writes for close at 1732293840882Writing region close event to WAL at 1732293840882Closed at 1732293840882 2024-11-22T16:44:00,885 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:44:00,890 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1732293840885"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293840885"}]},"ts":"1732293840885"} 2024-11-22T16:44:00,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:44:00,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:44:00,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293840899"}]},"ts":"1732293840899"} 2024-11-22T16:44:00,907 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLING in hbase:meta 2024-11-22T16:44:00,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, ASSIGN}] 2024-11-22T16:44:00,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, ASSIGN 2024-11-22T16:44:00,916 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, ASSIGN; state=OFFLINE, location=4618c6346924,46439,1732293836577; forceNewPlan=false, retain=false 2024-11-22T16:44:01,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=827a188ec32f56dcad90ad604a5da931, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:01,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, ASSIGN because future has completed 2024-11-22T16:44:01,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 827a188ec32f56dcad90ad604a5da931, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:01,239 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,239 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 827a188ec32f56dcad90ad604a5da931, NAME => 'testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:44:01,240 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTable 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,240 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:01,240 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,240 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,248 INFO [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,251 INFO [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 827a188ec32f56dcad90ad604a5da931 columnFamilyName f1 2024-11-22T16:44:01,251 DEBUG [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:01,253 INFO [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] regionserver.HStore(327): Store=827a188ec32f56dcad90ad604a5da931/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:01,253 INFO [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,256 INFO [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 827a188ec32f56dcad90ad604a5da931 columnFamilyName f2 2024-11-22T16:44:01,256 DEBUG [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:01,257 INFO [StoreOpener-827a188ec32f56dcad90ad604a5da931-1 {}] regionserver.HStore(327): Store=827a188ec32f56dcad90ad604a5da931/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:01,257 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,259 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,259 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,261 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,261 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,262 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTable descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T16:44:01,264 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,268 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:44:01,270 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 827a188ec32f56dcad90ad604a5da931; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67552606, jitterRate=0.006612271070480347}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T16:44:01,270 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,271 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 827a188ec32f56dcad90ad604a5da931: Running coprocessor pre-open hook at 1732293841241Writing region info on filesystem at 1732293841241Initializing all the Stores at 1732293841243 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293841243Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293841248 (+5 ms)Cleaning up temporary data from old regions at 1732293841261 (+13 ms)Running coprocessor post-open hooks at 1732293841270 (+9 ms)Region opened successfully at 1732293841271 (+1 ms) 2024-11-22T16:44:01,273 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931., pid=6, masterSystemTime=1732293841228 2024-11-22T16:44:01,276 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,277 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=827a188ec32f56dcad90ad604a5da931, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:01,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 827a188ec32f56dcad90ad604a5da931, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:01,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T16:44:01,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 827a188ec32f56dcad90ad604a5da931, server=4618c6346924,46439,1732293836577 in 210 msec 2024-11-22T16:44:01,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T16:44:01,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, ASSIGN in 380 msec 2024-11-22T16:44:01,296 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:44:01,297 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293841296"}]},"ts":"1732293841296"} 2024-11-22T16:44:01,300 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLED in hbase:meta 2024-11-22T16:44:01,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:44:01,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable in 1.2900 sec 2024-11-22T16:44:01,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:01,534 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293841534"}]},"ts":"1732293841534"} 2024-11-22T16:44:01,537 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLING in hbase:meta 2024-11-22T16:44:01,537 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTable to state=DISABLING 2024-11-22T16:44:01,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTable}] 2024-11-22T16:44:01,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, UNASSIGN}] 2024-11-22T16:44:01,550 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, UNASSIGN 2024-11-22T16:44:01,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=827a188ec32f56dcad90ad604a5da931, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:01,554 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:01,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, UNASSIGN because future has completed 2024-11-22T16:44:01,557 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:01,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 827a188ec32f56dcad90ad604a5da931, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:01,717 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,717 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T16:44:01,718 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 827a188ec32f56dcad90ad604a5da931, disabling compactions & flushes 2024-11-22T16:44:01,718 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,718 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,718 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. after waiting 0 ms 2024-11-22T16:44:01,718 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,726 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T16:44:01,729 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931. 2024-11-22T16:44:01,729 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 827a188ec32f56dcad90ad604a5da931: Waiting for close lock at 1732293841718Running coprocessor pre-close hooks at 1732293841718Disabling compacts and flushes for region at 1732293841718Disabling writes for close at 1732293841718Writing region close event to WAL at 1732293841718Running coprocessor post-close hooks at 1732293841727 (+9 ms)Closed at 1732293841729 (+2 ms) 2024-11-22T16:44:01,734 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,735 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=827a188ec32f56dcad90ad604a5da931, regionState=CLOSED 2024-11-22T16:44:01,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 827a188ec32f56dcad90ad604a5da931, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:01,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T16:44:01,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 827a188ec32f56dcad90ad604a5da931, server=4618c6346924,46439,1732293836577 in 184 msec 2024-11-22T16:44:01,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T16:44:01,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=827a188ec32f56dcad90ad604a5da931, UNASSIGN in 199 msec 2024-11-22T16:44:01,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T16:44:01,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTable in 212 msec 2024-11-22T16:44:01,759 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293841759"}]},"ts":"1732293841759"} 2024-11-22T16:44:01,762 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLED in hbase:meta 2024-11-22T16:44:01,763 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTable to state=DISABLED 2024-11-22T16:44:01,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable in 240 msec 2024-11-22T16:44:01,780 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T16:44:01,784 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T16:44:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4618c6346924,45935,1732293835695 2024-11-22T16:44:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ee78c4f 2024-11-22T16:44:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T16:44:01,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T16:44:01,801 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTable 2024-11-22T16:44:01,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTable 2024-11-22T16:44:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] ipc.CallRunner(138): callId: 2 service: MasterService methodName: DisableTable size: 78 connection: 172.17.0.2:45210 deadline: 1732293901804, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable 2024-11-22T16:44:01,832 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTable failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T16:44:01,832 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTable already disabled, so just deleting it. 2024-11-22T16:44:01,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTable 2024-11-22T16:44:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] procedure2.ProcedureExecutor(1139): Stored pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:01,848 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:01,851 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=12, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-22T16:44:01,860 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,867 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/f1, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/f2, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/recovered.edits] 2024-11-22T16:44:01,883 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/recovered.edits/4.seqid to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/archive/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931/recovered.edits/4.seqid 2024-11-22T16:44:01,884 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTable/827a188ec32f56dcad90ad604a5da931 2024-11-22T16:44:01,884 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTable regions 2024-11-22T16:44:01,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=12, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:01,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46439 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T16:44:01,904 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTable from hbase:meta 2024-11-22T16:44:01,942 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTable' descriptor. 2024-11-22T16:44:01,946 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=12, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:01,946 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTable' from region states. 2024-11-22T16:44:01,946 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732293841946"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:01,953 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T16:44:01,953 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 827a188ec32f56dcad90ad604a5da931, NAME => 'testDisableTable,,1732293840010.827a188ec32f56dcad90ad604a5da931.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T16:44:01,953 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTable' as deleted. 2024-11-22T16:44:01,954 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732293841953"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:01,958 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTable state from META 2024-11-22T16:44:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-22T16:44:01,960 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=12, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:01,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable in 120 msec 2024-11-22T16:44:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-22T16:44:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTable 2024-11-22T16:44:02,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTable completed 2024-11-22T16:44:02,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTable Thread=217 (was 207) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4618c6346924:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4618c6346924:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_896826000_22 at /127.0.0.1:59388 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 289), ProcessCount=11 (was 11), AvailableMemoryMB=8194 (was 8314) 2024-11-22T16:44:02,193 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=217, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=8194 2024-11-22T16:44:02,193 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T16:44:02,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:02,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:44:02,197 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:02,198 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:44:02,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741837_1013 (size=652) 2024-11-22T16:44:02,610 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21fe2786c1135acd8bdf640287743b2f, NAME => 'testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTableMultipleTimes', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:44:02,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741838_1014 (size=64) 2024-11-22T16:44:03,019 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:03,020 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1722): Closing 21fe2786c1135acd8bdf640287743b2f, disabling compactions & flushes 2024-11-22T16:44:03,020 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,020 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,020 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. after waiting 0 ms 2024-11-22T16:44:03,020 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,020 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,020 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21fe2786c1135acd8bdf640287743b2f: Waiting for close lock at 1732293843020Disabling compacts and flushes for region at 1732293843020Disabling writes for close at 1732293843020Writing region close event to WAL at 1732293843020Closed at 1732293843020 2024-11-22T16:44:03,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:44:03,023 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732293843023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293843023"}]},"ts":"1732293843023"} 2024-11-22T16:44:03,027 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:44:03,028 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:44:03,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293843029"}]},"ts":"1732293843029"} 2024-11-22T16:44:03,032 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLING in hbase:meta 2024-11-22T16:44:03,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, ASSIGN}] 2024-11-22T16:44:03,036 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, ASSIGN 2024-11-22T16:44:03,037 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, ASSIGN; state=OFFLINE, location=4618c6346924,46439,1732293836577; forceNewPlan=false, retain=false 2024-11-22T16:44:03,188 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=21fe2786c1135acd8bdf640287743b2f, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:03,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, ASSIGN because future has completed 2024-11-22T16:44:03,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21fe2786c1135acd8bdf640287743b2f, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:03,351 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,352 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 21fe2786c1135acd8bdf640287743b2f, NAME => 'testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:44:03,352 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTableMultipleTimes 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,352 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:03,352 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,353 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,355 INFO [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,357 INFO [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21fe2786c1135acd8bdf640287743b2f columnFamilyName f1 2024-11-22T16:44:03,357 DEBUG [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:03,358 INFO [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] regionserver.HStore(327): Store=21fe2786c1135acd8bdf640287743b2f/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:03,358 INFO [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,361 INFO [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21fe2786c1135acd8bdf640287743b2f columnFamilyName f2 2024-11-22T16:44:03,361 DEBUG [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:03,362 INFO [StoreOpener-21fe2786c1135acd8bdf640287743b2f-1 {}] regionserver.HStore(327): Store=21fe2786c1135acd8bdf640287743b2f/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:03,362 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,363 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,364 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,365 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,365 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,366 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTableMultipleTimes descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T16:44:03,368 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,371 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:44:03,372 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 21fe2786c1135acd8bdf640287743b2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74539791, jitterRate=0.11072944104671478}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T16:44:03,372 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,373 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 21fe2786c1135acd8bdf640287743b2f: Running coprocessor pre-open hook at 1732293843353Writing region info on filesystem at 1732293843353Initializing all the Stores at 1732293843354 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293843354Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293843354Cleaning up temporary data from old regions at 1732293843365 (+11 ms)Running coprocessor post-open hooks at 1732293843372 (+7 ms)Region opened successfully at 1732293843373 (+1 ms) 2024-11-22T16:44:03,374 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f., pid=15, masterSystemTime=1732293843346 2024-11-22T16:44:03,378 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,378 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=21fe2786c1135acd8bdf640287743b2f, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:03,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21fe2786c1135acd8bdf640287743b2f, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:03,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-22T16:44:03,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 21fe2786c1135acd8bdf640287743b2f, server=4618c6346924,46439,1732293836577 in 192 msec 2024-11-22T16:44:03,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T16:44:03,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, ASSIGN in 355 msec 2024-11-22T16:44:03,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:44:03,391 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293843391"}]},"ts":"1732293843391"} 2024-11-22T16:44:03,393 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLED in hbase:meta 2024-11-22T16:44:03,395 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:44:03,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes in 1.2020 sec 2024-11-22T16:44:03,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,452 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293843451"}]},"ts":"1732293843451"} 2024-11-22T16:44:03,455 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLING in hbase:meta 2024-11-22T16:44:03,455 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTableMultipleTimes to state=DISABLING 2024-11-22T16:44:03,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes}] 2024-11-22T16:44:03,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, UNASSIGN}] 2024-11-22T16:44:03,461 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, UNASSIGN 2024-11-22T16:44:03,462 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=21fe2786c1135acd8bdf640287743b2f, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:03,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, UNASSIGN because future has completed 2024-11-22T16:44:03,465 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:03,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21fe2786c1135acd8bdf640287743b2f, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:03,619 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(122): Close 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,619 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T16:44:03,619 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1722): Closing 21fe2786c1135acd8bdf640287743b2f, disabling compactions & flushes 2024-11-22T16:44:03,619 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,619 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,619 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. after waiting 0 ms 2024-11-22T16:44:03,619 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,626 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T16:44:03,627 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f. 2024-11-22T16:44:03,627 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1676): Region close journal for 21fe2786c1135acd8bdf640287743b2f: Waiting for close lock at 1732293843619Running coprocessor pre-close hooks at 1732293843619Disabling compacts and flushes for region at 1732293843619Disabling writes for close at 1732293843619Writing region close event to WAL at 1732293843620 (+1 ms)Running coprocessor post-close hooks at 1732293843626 (+6 ms)Closed at 1732293843626 2024-11-22T16:44:03,630 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(157): Closed 21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=21fe2786c1135acd8bdf640287743b2f, regionState=CLOSED 2024-11-22T16:44:03,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21fe2786c1135acd8bdf640287743b2f, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:03,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-22T16:44:03,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; CloseRegionProcedure 21fe2786c1135acd8bdf640287743b2f, server=4618c6346924,46439,1732293836577 in 171 msec 2024-11-22T16:44:03,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T16:44:03,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=21fe2786c1135acd8bdf640287743b2f, UNASSIGN in 180 msec 2024-11-22T16:44:03,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-22T16:44:03,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes in 187 msec 2024-11-22T16:44:03,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293843648"}]},"ts":"1732293843648"} 2024-11-22T16:44:03,651 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLED in hbase:meta 2024-11-22T16:44:03,651 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTableMultipleTimes to state=DISABLED 2024-11-22T16:44:03,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes in 206 msec 2024-11-22T16:44:03,698 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(101): Disable failed with exception {}org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes 2024-11-22T16:44:03,699 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(114): Disable failed with expected exception {} org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:109) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:03,699 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(126): Disable failed with expected exception {} org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:64) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:120) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:03,699 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T16:44:03,702 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTableMultipleTimes 2024-11-22T16:44:03,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTableMultipleTimes 2024-11-22T16:44:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] ipc.CallRunner(138): callId: 8 service: MasterService methodName: DisableTable size: 90 connection: 172.17.0.2:45210 deadline: 1732293903702, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes 2024-11-22T16:44:03,703 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTableMultipleTimes failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T16:44:03,703 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTableMultipleTimes already disabled, so just deleting it. 2024-11-22T16:44:03,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTableMultipleTimes 2024-11-22T16:44:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,707 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,708 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T16:44:03,713 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,717 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/f1, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/f2, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/recovered.edits] 2024-11-22T16:44:03,728 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/recovered.edits/4.seqid to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/archive/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f/recovered.edits/4.seqid 2024-11-22T16:44:03,729 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testDisableTableMultipleTimes/21fe2786c1135acd8bdf640287743b2f 2024-11-22T16:44:03,729 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTableMultipleTimes regions 2024-11-22T16:44:03,733 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,736 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTableMultipleTimes from hbase:meta 2024-11-22T16:44:03,740 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTableMultipleTimes' descriptor. 2024-11-22T16:44:03,742 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,742 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTableMultipleTimes' from region states. 2024-11-22T16:44:03,743 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732293843742"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:03,746 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T16:44:03,746 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21fe2786c1135acd8bdf640287743b2f, NAME => 'testDisableTableMultipleTimes,,1732293842193.21fe2786c1135acd8bdf640287743b2f.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T16:44:03,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTableMultipleTimes' as deleted. 2024-11-22T16:44:03,747 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732293843747"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:03,751 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTableMultipleTimes state from META 2024-11-22T16:44:03,753 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:03,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes in 49 msec 2024-11-22T16:44:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T16:44:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTableMultipleTimes 2024-11-22T16:44:03,823 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTableMultipleTimes completed 2024-11-22T16:44:03,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=224 (was 217) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_896826000_22 at /127.0.0.1:59388 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 289), ProcessCount=11 (was 11), AvailableMemoryMB=8171 (was 8194) 2024-11-22T16:44:03,844 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=224, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=8171 2024-11-22T16:44:03,844 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T16:44:03,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:03,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:44:03,850 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:03,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:44:03,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741839_1015 (size=653) 2024-11-22T16:44:03,867 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e431292ce15de70c4d2886b2a2b62d3, NAME => 'testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:44:03,868 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => afb9e4c8ad52d77a236d6d40b38f107b, NAME => 'testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:44:03,868 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 08d895a44bedb76f7caed87c8c9bdcf0, NAME => 'testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:44:03,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741840_1016 (size=66) 2024-11-22T16:44:03,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741841_1017 (size=67) 2024-11-22T16:44:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741842_1018 (size=67) 2024-11-22T16:44:04,271 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T16:44:04,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:04,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing 1e431292ce15de70c4d2886b2a2b62d3, disabling compactions & flushes 2024-11-22T16:44:04,281 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:04,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:04,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. after waiting 0 ms 2024-11-22T16:44:04,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:04,281 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:04,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e431292ce15de70c4d2886b2a2b62d3: Waiting for close lock at 1732293844281Disabling compacts and flushes for region at 1732293844281Disabling writes for close at 1732293844281Writing region close event to WAL at 1732293844281Closed at 1732293844281 2024-11-22T16:44:04,282 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42d7a960e5facf87ab193a07a29414cc, NAME => 'testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.', STARTKEY => 'c', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1 2024-11-22T16:44:04,284 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:04,284 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1722): Closing afb9e4c8ad52d77a236d6d40b38f107b, disabling compactions & flushes 2024-11-22T16:44:04,284 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:04,284 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:04,284 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. after waiting 0 ms 2024-11-22T16:44:04,284 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:04,284 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:04,284 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1676): Region close journal for afb9e4c8ad52d77a236d6d40b38f107b: Waiting for close lock at 1732293844284Disabling compacts and flushes for region at 1732293844284Disabling writes for close at 1732293844284Writing region close event to WAL at 1732293844284Closed at 1732293844284 2024-11-22T16:44:04,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741843_1019 (size=66) 2024-11-22T16:44:04,291 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:04,291 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 08d895a44bedb76f7caed87c8c9bdcf0, disabling compactions & flushes 2024-11-22T16:44:04,291 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:04,291 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:04,291 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. after waiting 0 ms 2024-11-22T16:44:04,291 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:04,291 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:04,292 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 08d895a44bedb76f7caed87c8c9bdcf0: Waiting for close lock at 1732293844291Disabling compacts and flushes for region at 1732293844291Disabling writes for close at 1732293844291Writing region close event to WAL at 1732293844291Closed at 1732293844291 2024-11-22T16:44:04,691 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:04,691 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing 42d7a960e5facf87ab193a07a29414cc, disabling compactions & flushes 2024-11-22T16:44:04,691 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:04,691 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:04,691 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. after waiting 0 ms 2024-11-22T16:44:04,691 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:04,691 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:04,691 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42d7a960e5facf87ab193a07a29414cc: Waiting for close lock at 1732293844691Disabling compacts and flushes for region at 1732293844691Disabling writes for close at 1732293844691Writing region close event to WAL at 1732293844691Closed at 1732293844691 2024-11-22T16:44:04,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:44:04,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1732293844694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293844694"}]},"ts":"1732293844694"} 2024-11-22T16:44:04,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732293844694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293844694"}]},"ts":"1732293844694"} 2024-11-22T16:44:04,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732293844694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293844694"}]},"ts":"1732293844694"} 2024-11-22T16:44:04,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1732293844694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732293844694"}]},"ts":"1732293844694"} 2024-11-22T16:44:04,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 4 regions to meta. 2024-11-22T16:44:04,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:44:04,703 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293844703"}]},"ts":"1732293844703"} 2024-11-22T16:44:04,706 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLING in hbase:meta 2024-11-22T16:44:04,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, ASSIGN}, {pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, ASSIGN}, {pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, ASSIGN}, {pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, ASSIGN}] 2024-11-22T16:44:04,710 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, ASSIGN 2024-11-22T16:44:04,711 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, ASSIGN; state=OFFLINE, location=4618c6346924,46439,1732293836577; forceNewPlan=false, retain=false 2024-11-22T16:44:04,712 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, ASSIGN 2024-11-22T16:44:04,713 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, ASSIGN; state=OFFLINE, location=4618c6346924,46439,1732293836577; forceNewPlan=false, retain=false 2024-11-22T16:44:04,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, ASSIGN 2024-11-22T16:44:04,715 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, ASSIGN; state=OFFLINE, location=4618c6346924,46439,1732293836577; forceNewPlan=false, retain=false 2024-11-22T16:44:04,717 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, ASSIGN 2024-11-22T16:44:04,718 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, ASSIGN; state=OFFLINE, location=4618c6346924,46439,1732293836577; forceNewPlan=false, retain=false 2024-11-22T16:44:04,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:04,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:04,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:04,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPENING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:04,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, ASSIGN because future has completed 2024-11-22T16:44:04,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:04,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, ASSIGN because future has completed 2024-11-22T16:44:04,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, ASSIGN because future has completed 2024-11-22T16:44:04,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, ASSIGN because future has completed 2024-11-22T16:44:04,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:04,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:04,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:05,027 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:05,028 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 42d7a960e5facf87ab193a07a29414cc, NAME => 'testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.', STARTKEY => 'c', ENDKEY => ''} 2024-11-22T16:44:05,028 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,028 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:05,028 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,028 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,030 INFO [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,032 INFO [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42d7a960e5facf87ab193a07a29414cc columnFamilyName f1 2024-11-22T16:44:05,032 DEBUG [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,033 INFO [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] regionserver.HStore(327): Store=42d7a960e5facf87ab193a07a29414cc/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,033 INFO [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,035 INFO [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42d7a960e5facf87ab193a07a29414cc columnFamilyName f2 2024-11-22T16:44:05,035 DEBUG [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,036 INFO [StoreOpener-42d7a960e5facf87ab193a07a29414cc-1 {}] regionserver.HStore(327): Store=42d7a960e5facf87ab193a07a29414cc/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,036 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,037 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,038 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,039 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,039 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,040 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T16:44:05,041 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,044 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:44:05,045 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1114): Opened 42d7a960e5facf87ab193a07a29414cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71927059, jitterRate=0.07179670035839081}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T16:44:05,045 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:05,046 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 42d7a960e5facf87ab193a07a29414cc: Running coprocessor pre-open hook at 1732293845028Writing region info on filesystem at 1732293845028Initializing all the Stores at 1732293845030 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845030Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845030Cleaning up temporary data from old regions at 1732293845039 (+9 ms)Running coprocessor post-open hooks at 1732293845045 (+6 ms)Region opened successfully at 1732293845046 (+1 ms) 2024-11-22T16:44:05,048 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc., pid=27, masterSystemTime=1732293845021 2024-11-22T16:44:05,052 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:05,052 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:05,052 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:05,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:05,052 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e431292ce15de70c4d2886b2a2b62d3, NAME => 'testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.', STARTKEY => '', ENDKEY => 'a'} 2024-11-22T16:44:05,053 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,053 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:05,053 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,053 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:05,055 INFO [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,058 INFO [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e431292ce15de70c4d2886b2a2b62d3 columnFamilyName f1 2024-11-22T16:44:05,058 DEBUG [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,059 INFO [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] regionserver.HStore(327): Store=1e431292ce15de70c4d2886b2a2b62d3/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,060 INFO [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-22T16:44:05,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 in 191 msec 2024-11-22T16:44:05,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, ASSIGN in 354 msec 2024-11-22T16:44:05,064 INFO [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e431292ce15de70c4d2886b2a2b62d3 columnFamilyName f2 2024-11-22T16:44:05,064 DEBUG [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,065 INFO [StoreOpener-1e431292ce15de70c4d2886b2a2b62d3-1 {}] regionserver.HStore(327): Store=1e431292ce15de70c4d2886b2a2b62d3/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,065 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,066 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,067 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,069 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,069 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,071 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:44:05,076 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T16:44:05,078 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,090 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:44:05,091 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 1e431292ce15de70c4d2886b2a2b62d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72776559, jitterRate=0.08445523679256439}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T16:44:05,091 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:05,091 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 1e431292ce15de70c4d2886b2a2b62d3: Running coprocessor pre-open hook at 1732293845053Writing region info on filesystem at 1732293845053Initializing all the Stores at 1732293845055 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845055Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845055Cleaning up temporary data from old regions at 1732293845069 (+14 ms)Running coprocessor post-open hooks at 1732293845091 (+22 ms)Region opened successfully at 1732293845091 2024-11-22T16:44:05,093 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3., pid=28, masterSystemTime=1732293845021 2024-11-22T16:44:05,097 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:05,097 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:05,097 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:05,098 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => 08d895a44bedb76f7caed87c8c9bdcf0, NAME => 'testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.', STARTKEY => 'a', ENDKEY => 'b'} 2024-11-22T16:44:05,098 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,098 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:05,098 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:05,098 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,098 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:05,103 INFO [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,107 INFO [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d895a44bedb76f7caed87c8c9bdcf0 columnFamilyName f1 2024-11-22T16:44:05,107 DEBUG [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=23 2024-11-22T16:44:05,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 in 230 msec 2024-11-22T16:44:05,108 INFO [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] regionserver.HStore(327): Store=08d895a44bedb76f7caed87c8c9bdcf0/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,108 INFO [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, ASSIGN in 400 msec 2024-11-22T16:44:05,111 INFO [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d895a44bedb76f7caed87c8c9bdcf0 columnFamilyName f2 2024-11-22T16:44:05,111 DEBUG [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,112 INFO [StoreOpener-08d895a44bedb76f7caed87c8c9bdcf0-1 {}] regionserver.HStore(327): Store=08d895a44bedb76f7caed87c8c9bdcf0/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,112 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,114 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,116 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,117 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,117 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,118 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T16:44:05,121 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,125 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:44:05,126 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened 08d895a44bedb76f7caed87c8c9bdcf0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61562534, jitterRate=-0.08264675736427307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T16:44:05,126 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:05,127 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for 08d895a44bedb76f7caed87c8c9bdcf0: Running coprocessor pre-open hook at 1732293845099Writing region info on filesystem at 1732293845099Initializing all the Stores at 1732293845101 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845101Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845102 (+1 ms)Cleaning up temporary data from old regions at 1732293845117 (+15 ms)Running coprocessor post-open hooks at 1732293845127 (+10 ms)Region opened successfully at 1732293845127 2024-11-22T16:44:05,128 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0., pid=29, masterSystemTime=1732293845021 2024-11-22T16:44:05,132 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:05,132 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:05,132 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:05,132 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => afb9e4c8ad52d77a236d6d40b38f107b, NAME => 'testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.', STARTKEY => 'b', ENDKEY => 'c'} 2024-11-22T16:44:05,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:05,133 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,133 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:44:05,133 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,133 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,136 INFO [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:05,139 INFO [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afb9e4c8ad52d77a236d6d40b38f107b columnFamilyName f1 2024-11-22T16:44:05,139 DEBUG [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,140 INFO [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] regionserver.HStore(327): Store=afb9e4c8ad52d77a236d6d40b38f107b/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,140 INFO [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=24 2024-11-22T16:44:05,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 in 260 msec 2024-11-22T16:44:05,142 INFO [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afb9e4c8ad52d77a236d6d40b38f107b columnFamilyName f2 2024-11-22T16:44:05,142 DEBUG [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:44:05,143 INFO [StoreOpener-afb9e4c8ad52d77a236d6d40b38f107b-1 {}] regionserver.HStore(327): Store=afb9e4c8ad52d77a236d6d40b38f107b/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:44:05,143 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, ASSIGN in 435 msec 2024-11-22T16:44:05,145 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,146 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,147 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,147 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,148 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T16:44:05,150 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,153 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:44:05,154 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened afb9e4c8ad52d77a236d6d40b38f107b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65867481, jitterRate=-0.018498048186302185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T16:44:05,154 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:05,154 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for afb9e4c8ad52d77a236d6d40b38f107b: Running coprocessor pre-open hook at 1732293845133Writing region info on filesystem at 1732293845133Initializing all the Stores at 1732293845135 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845135Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732293845135Cleaning up temporary data from old regions at 1732293845147 (+12 ms)Running coprocessor post-open hooks at 1732293845154 (+7 ms)Region opened successfully at 1732293845154 2024-11-22T16:44:05,155 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b., pid=30, masterSystemTime=1732293845021 2024-11-22T16:44:05,159 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:05,159 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:05,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:05,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:05,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=25 2024-11-22T16:44:05,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 in 282 msec 2024-11-22T16:44:05,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=22 2024-11-22T16:44:05,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, ASSIGN in 461 msec 2024-11-22T16:44:05,173 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:44:05,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293845173"}]},"ts":"1732293845173"} 2024-11-22T16:44:05,176 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLED in hbase:meta 2024-11-22T16:44:05,177 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:44:05,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution in 1.3320 sec 2024-11-22T16:44:05,348 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-22T16:44:05,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:05,352 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:05,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:05,601 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,602 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:05,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:05,602 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:05,602 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:05,602 ERROR [Time-limited test {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] 2024-11-22T16:44:05,603 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:05,603 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:05,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:05,604 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:05,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:05,611 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[] 2024-11-22T16:44:05,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:05,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:05,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:05,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:05,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:05,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:05,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:05,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:05,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-22T16:44:05,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,620 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:05,621 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:05,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:05,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:05,626 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293875625 2024-11-22T16:44:05,626 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:05,626 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,635 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:05,635 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293905635 2024-11-22T16:44:05,635 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293965635 2024-11-22T16:44:05,635 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T16:44:05,635 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:05,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:05,637 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:05,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:05,885 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,885 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:05,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:05,886 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:05,886 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:05,886 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:05,886 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:05,886 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:05,887 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:05,888 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:05,891 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0] 2024-11-22T16:44:05,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:05,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:05,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:05,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:05,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:05,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:05,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:05,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:05,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:05,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,895 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:05,895 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:05,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:05,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:05,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:05,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:05,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293875900 2024-11-22T16:44:05,900 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:05,900 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:05,906 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:05,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293905906 2024-11-22T16:44:05,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293965906 2024-11-22T16:44:05,906 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:05,906 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:05,908 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293845908"}]},"ts":"1732293845908"} 2024-11-22T16:44:05,911 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta 2024-11-22T16:44:05,911 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING 2024-11-22T16:44:05,911 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:05,911 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:05,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:05,911 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:06,156 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,156 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:06,156 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,156 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:06,156 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:06,157 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:06,157 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:06,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:06,157 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:06,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:06,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1] 2024-11-22T16:44:06,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:06,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:06,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:06,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:06,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,163 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:06,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:44:06,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,164 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:06,165 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:06,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,168 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293876168 2024-11-22T16:44:06,168 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:06,168 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,173 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:06,173 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293906173 2024-11-22T16:44:06,173 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293966173 2024-11-22T16:44:06,173 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:06,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293846173"}]},"ts":"1732293846173"} 2024-11-22T16:44:06,176 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta 2024-11-22T16:44:06,176 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING 2024-11-22T16:44:06,176 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:06,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}] 2024-11-22T16:44:06,177 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:06,177 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:06,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T16:44:06,194 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T16:44:06,195 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:44:06,195 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T16:44:06,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T16:44:06,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T16:44:06,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution 2024-11-22T16:44:06,197 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution Metrics about Tables on a single HBase RegionServer 2024-11-22T16:44:06,423 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,423 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:06,423 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,423 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:06,423 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:06,424 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:06,424 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:06,424 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:06,424 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:06,425 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:06,428 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2] 2024-11-22T16:44:06,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:06,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:06,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:06,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:06,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,430 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:06,430 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T16:44:06,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,432 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:06,432 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:06,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,436 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,436 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,437 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293876436 2024-11-22T16:44:06,437 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:06,437 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,442 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:06,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293906442 2024-11-22T16:44:06,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293966442 2024-11-22T16:44:06,442 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:06,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}] 2024-11-22T16:44:06,442 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:06,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN}] 2024-11-22T16:44:06,446 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:06,446 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,447 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:06,692 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,692 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:06,692 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,692 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:06,692 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:06,693 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:06,693 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:06,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:06,693 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:06,694 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T16:44:06,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:06,696 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[] 2024-11-22T16:44:06,696 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:06,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:06,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:06,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:06,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:06,699 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:06,699 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T16:44:06,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,700 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:06,700 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:06,701 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:06,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,703 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293876703 2024-11-22T16:44:06,703 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:06,703 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,709 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,709 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:06,709 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293906709 2024-11-22T16:44:06,709 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293966709 2024-11-22T16:44:06,709 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:06,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN}] 2024-11-22T16:44:06,710 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:06,713 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:06,714 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,715 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:06,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:06,717 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:06,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,720 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:06,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:06,720 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:06,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:06,720 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:06,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,720 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:06,720 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fe21cbd36e0@3b70dc72 rejected from java.util.concurrent.ThreadPoolExecutor@792ded24[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:06,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN because future has completed 2024-11-22T16:44:06,722 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fe21cbd36e0@601e77b1 rejected from java.util.concurrent.ThreadPoolExecutor@792ded24[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:06,806 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:06,806 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:06,908 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:06,909 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:06,959 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,959 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:06,959 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,960 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:06,960 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:06,960 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:06,960 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:06,960 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:06,960 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:06,961 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:06,964 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:06,965 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:06,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:06,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:06,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:06,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,968 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,968 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:06,969 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:06,969 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:06,969 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:06,969 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:06,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-22T16:44:06,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,971 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:06,971 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:06,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:06,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:06,974 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3 to restore RIT 2024-11-22T16:44:06,974 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0 to restore RIT 2024-11-22T16:44:06,974 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:06,974 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:06,974 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293876974 2024-11-22T16:44:06,975 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:06,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:06,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:06,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:06,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:06,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:06,980 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:06,980 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293906980 2024-11-22T16:44:06,980 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293966980 2024-11-22T16:44:06,980 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:06,980 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:06,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:06,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN because future has completed 2024-11-22T16:44:06,985 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:06,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:06,985 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:06,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN because future has completed 2024-11-22T16:44:06,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:06,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:06,989 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:06,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:06,990 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:06,990 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:06,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:06,990 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:07,011 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,012 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,114 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,115 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,139 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(122): Close 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:07,139 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T16:44:07,139 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1722): Closing 1e431292ce15de70c4d2886b2a2b62d3, disabling compactions & flushes 2024-11-22T16:44:07,139 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:07,139 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:07,139 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. after waiting 0 ms 2024-11-22T16:44:07,139 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:07,145 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T16:44:07,146 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3. 2024-11-22T16:44:07,146 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1676): Region close journal for 1e431292ce15de70c4d2886b2a2b62d3: Waiting for close lock at 1732293847139Running coprocessor pre-close hooks at 1732293847139Disabling compacts and flushes for region at 1732293847139Disabling writes for close at 1732293847139Writing region close event to WAL at 1732293847140 (+1 ms)Running coprocessor post-close hooks at 1732293847146 (+6 ms)Closed at 1732293847146 2024-11-22T16:44:07,149 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(157): Closed 1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:07,217 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,218 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,230 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,230 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:07,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:07,230 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:07,230 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:07,231 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:07,231 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:07,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:07,231 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:07,232 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6] 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:07,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:07,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:07,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:07,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,238 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,238 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,238 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:07,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,239 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:07,239 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:07,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:07,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:07,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:07,242 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:07,242 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3 to restore RIT 2024-11-22T16:44:07,242 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0 to restore RIT 2024-11-22T16:44:07,242 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:07,242 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:07,243 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293877243 2024-11-22T16:44:07,243 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:07,243 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,243 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,243 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,243 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,248 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:07,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293907248 2024-11-22T16:44:07,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293967248 2024-11-22T16:44:07,248 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:07,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED 2024-11-22T16:44:07,252 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:07,252 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,254 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:07,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN because future has completed 2024-11-22T16:44:07,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:07,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=33 2024-11-22T16:44:07,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 in 269 msec 2024-11-22T16:44:07,258 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:07,258 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:07,258 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:07,321 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,321 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,424 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,424 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,498 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,498 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:07,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:07,498 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:07,499 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:07,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:07,499 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:07,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:07,499 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:07,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,502 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:07,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:07,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:07,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:07,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,506 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:07,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,507 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:07,507 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:07,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:07,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:07,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:07,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:07,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3 to restore RIT 2024-11-22T16:44:07,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0 to restore RIT 2024-11-22T16:44:07,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:07,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:07,515 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293877515 2024-11-22T16:44:07,516 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:07,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,516 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,520 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:07,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293907520 2024-11-22T16:44:07,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293967520 2024-11-22T16:44:07,520 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:07,521 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:07,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN in 813 msec 2024-11-22T16:44:07,524 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN because future has completed 2024-11-22T16:44:07,531 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:07,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:07,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:07,531 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:07,531 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:07,531 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:07,532 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fe21cbd36e0@5dbd00f rejected from java.util.concurrent.ThreadPoolExecutor@492fff46[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:07,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:07,533 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fe21cbd36e0@77af07cd rejected from java.util.concurrent.ThreadPoolExecutor@492fff46[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:07,635 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,635 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,738 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,738 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,771 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,771 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:07,771 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:07,771 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:07,771 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:07,771 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:07,771 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:07,772 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:07,772 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:07,773 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:07,776 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:07,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:07,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:07,777 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:07,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:07,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,780 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,780 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,780 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:07,780 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,781 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:07,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T16:44:07,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,785 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:07,785 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:07,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:07,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:07,787 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:07,787 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:07,788 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0 to restore RIT 2024-11-22T16:44:07,788 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:07,788 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:07,791 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293877791 2024-11-22T16:44:07,791 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:07,791 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:07,791 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:07,791 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:07,801 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:07,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293907801 2024-11-22T16:44:07,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293967801 2024-11-22T16:44:07,801 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-22T16:44:07,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,814 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN because future has completed 2024-11-22T16:44:07,818 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:07,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:07,821 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:07,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:07,822 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:07,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:07,827 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:07,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:07,827 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:07,827 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:07,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:07,827 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:07,841 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,944 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:07,976 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:07,976 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T16:44:07,976 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 08d895a44bedb76f7caed87c8c9bdcf0, disabling compactions & flushes 2024-11-22T16:44:07,976 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:07,976 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:07,976 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. after waiting 0 ms 2024-11-22T16:44:07,976 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:07,983 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T16:44:07,984 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0. 2024-11-22T16:44:07,984 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 08d895a44bedb76f7caed87c8c9bdcf0: Waiting for close lock at 1732293847976Running coprocessor pre-close hooks at 1732293847976Disabling compacts and flushes for region at 1732293847976Disabling writes for close at 1732293847976Writing region close event to WAL at 1732293847977 (+1 ms)Running coprocessor post-close hooks at 1732293847984 (+7 ms)Closed at 1732293847984 2024-11-22T16:44:07,987 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:08,047 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,051 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 9 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,051 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:08,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,052 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:08,052 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:08,052 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:08,052 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:08,052 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:08,052 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:08,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:08,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:08,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:08,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:08,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:08,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,061 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:08,061 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:08,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,063 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,063 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,064 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0 to restore RIT 2024-11-22T16:44:08,064 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:08,064 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:08,068 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293878068 2024-11-22T16:44:08,068 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:08,068 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,068 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,068 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,073 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:08,073 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293908073 2024-11-22T16:44:08,073 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293968073 2024-11-22T16:44:08,073 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:08,075 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED 2024-11-22T16:44:08,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:08,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:08,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:08,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=34 2024-11-22T16:44:08,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 in 259 msec 2024-11-22T16:44:08,083 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:08,083 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,083 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:08,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:08,085 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fe21cbd36e0@3f3e8b41 rejected from java.util.concurrent.ThreadPoolExecutor@1785b9cc[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 2] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:08,150 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,150 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,253 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,253 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,323 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 10 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,323 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:08,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,324 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:08,324 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:08,324 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:08,324 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:08,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:08,324 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:08,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:08,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:08,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:08,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:08,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,331 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:44:08,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,332 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:08,332 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:08,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,336 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,336 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0 to restore RIT 2024-11-22T16:44:08,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:08,337 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:08,337 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293878337 2024-11-22T16:44:08,337 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:08,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,341 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:08,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293908341 2024-11-22T16:44:08,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293968341 2024-11-22T16:44:08,341 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:08,341 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:08,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN in 1.6340 sec 2024-11-22T16:44:08,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:08,349 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:08,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:08,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:08,349 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:08,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:08,350 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fe21cbd36e0@4c73189 rejected from java.util.concurrent.ThreadPoolExecutor@5cd2187a[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:44:08,355 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,457 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,561 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:08,592 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 11 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,592 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:08,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,592 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:08,592 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:08,592 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:08,592 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:08,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:08,593 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:08,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:08,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:08,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:08,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:08,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:08,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,599 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,599 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,599 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:44:08,600 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,600 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:08,600 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,602 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,603 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:08,603 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:08,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293878603 2024-11-22T16:44:08,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:08,604 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,604 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,608 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:08,608 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293908608 2024-11-22T16:44:08,608 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293968608 2024-11-22T16:44:08,608 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:08,609 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,609 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN because future has completed 2024-11-22T16:44:08,612 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:08,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:08,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:08,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:08,617 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:08,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:08,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:08,617 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:08,767 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(122): Close afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:08,767 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T16:44:08,767 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1722): Closing afb9e4c8ad52d77a236d6d40b38f107b, disabling compactions & flushes 2024-11-22T16:44:08,767 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:08,767 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:08,767 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. after waiting 0 ms 2024-11-22T16:44:08,767 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:08,775 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T16:44:08,777 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b. 2024-11-22T16:44:08,777 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1676): Region close journal for afb9e4c8ad52d77a236d6d40b38f107b: Waiting for close lock at 1732293848767Running coprocessor pre-close hooks at 1732293848767Disabling compacts and flushes for region at 1732293848767Disabling writes for close at 1732293848767Writing region close event to WAL at 1732293848768 (+1 ms)Running coprocessor post-close hooks at 1732293848777 (+9 ms)Closed at 1732293848777 2024-11-22T16:44:08,780 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(157): Closed afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:08,859 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 12 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,859 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:08,859 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,859 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:08,859 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:08,859 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:08,859 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:08,859 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:08,860 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:08,861 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14] 2024-11-22T16:44:08,862 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:08,863 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:08,863 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:08,863 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:08,863 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:08,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:08,864 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,865 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:08,866 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:08,866 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:44:08,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,867 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:08,867 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:08,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:08,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:08,869 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:08,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:08,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:08,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293878870 2024-11-22T16:44:08,870 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:08,871 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,871 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:08,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:08,875 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:08,875 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293908875 2024-11-22T16:44:08,875 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293968875 2024-11-22T16:44:08,875 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:08,877 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED 2024-11-22T16:44:08,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:08,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:08,881 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:08,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:08,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=35 2024-11-22T16:44:08,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 in 269 msec 2024-11-22T16:44:08,885 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:08,885 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:08,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:08,885 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:08,973 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:09,076 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=4618c6346924,46439,1732293836577, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T16:44:09,126 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 13 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,126 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:09,126 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,126 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:09,126 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:09,127 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:09,127 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:09,127 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:09,127 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:09,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:09,130 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13] 2024-11-22T16:44:09,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:09,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:09,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:09,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:09,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:09,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:09,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:09,136 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T16:44:09,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,137 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:09,138 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:09,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b to restore RIT 2024-11-22T16:44:09,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:09,141 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293879141 2024-11-22T16:44:09,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:09,142 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:09,142 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,145 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:09,145 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293909145 2024-11-22T16:44:09,145 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293969145 2024-11-22T16:44:09,145 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:09,146 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:09,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:09,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN in 2.4380 sec 2024-11-22T16:44:09,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:09,152 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:09,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:09,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:09,152 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,152 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:09,396 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 14 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,396 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:09,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,396 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:09,396 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:09,396 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:09,396 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:09,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:09,397 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:09,398 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:09,400 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T16:44:09,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:09,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:09,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:09,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:09,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,405 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:09,406 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T16:44:09,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,407 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:09,407 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:09,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,411 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:09,412 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293879412 2024-11-22T16:44:09,412 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:09,412 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,417 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:09,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293909417 2024-11-22T16:44:09,418 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293969417 2024-11-22T16:44:09,418 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:09,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSING, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN because future has completed 2024-11-22T16:44:09,422 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T16:44:09,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577}] 2024-11-22T16:44:09,422 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:09,576 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:09,576 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T16:44:09,576 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 42d7a960e5facf87ab193a07a29414cc, disabling compactions & flushes 2024-11-22T16:44:09,577 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:09,577 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:09,577 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. after waiting 0 ms 2024-11-22T16:44:09,577 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:09,584 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T16:44:09,585 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc. 2024-11-22T16:44:09,586 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 42d7a960e5facf87ab193a07a29414cc: Waiting for close lock at 1732293849576Running coprocessor pre-close hooks at 1732293849576Disabling compacts and flushes for region at 1732293849576Disabling writes for close at 1732293849577 (+1 ms)Writing region close event to WAL at 1732293849577Running coprocessor post-close hooks at 1732293849585 (+8 ms)Closed at 1732293849585 2024-11-22T16:44:09,589 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:09,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED 2024-11-22T16:44:09,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:09,594 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:09,594 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,594 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:09,668 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 15 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,668 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:09,668 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,668 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:09,668 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:09,669 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:09,669 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:09,669 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:09,669 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:09,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17] 2024-11-22T16:44:09,672 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18] 2024-11-22T16:44:09,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:09,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:09,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:09,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:09,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:09,678 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,678 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:09,678 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T16:44:09,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,679 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:09,679 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:09,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,686 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:09,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293879686 2024-11-22T16:44:09,687 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:09,687 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,692 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:09,692 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293909692 2024-11-22T16:44:09,692 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293969692 2024-11-22T16:44:09,692 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:09,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, regionLocation=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 because future has completed 2024-11-22T16:44:09,698 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:09,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=36 2024-11-22T16:44:09,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 in 276 msec 2024-11-22T16:44:09,702 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:09,702 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,702 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:09,942 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 16 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,943 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:09,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,943 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:09,943 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:09,943 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:09,943 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:09,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:09,944 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:09,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17] 2024-11-22T16:44:09,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T16:44:09,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:09,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:09,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:09,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:09,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:09,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,951 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:09,952 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:09,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,952 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:09,953 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:09,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:09,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:09,955 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc to restore RIT 2024-11-22T16:44:09,956 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293879956 2024-11-22T16:44:09,956 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:09,956 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,959 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:09,960 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:09,960 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293909960 2024-11-22T16:44:09,960 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293969960 2024-11-22T16:44:09,960 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T16:44:09,960 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:09,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:09,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=32 2024-11-22T16:44:09,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN in 3.2530 sec 2024-11-22T16:44:09,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:09,966 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:09,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:09,966 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:10,210 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 17 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,210 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:10,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:10,210 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:10,211 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:10,211 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:10,211 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:10,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:10,211 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:10,212 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:10,214 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T16:44:10,215 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T16:44:10,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:10,215 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:10,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:10,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:10,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,216 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:10,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:10,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,220 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:10,220 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:10,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:10,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:10,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:10,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:10,224 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293880224 2024-11-22T16:44:10,224 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,224 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:10,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,228 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:10,228 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293910228 2024-11-22T16:44:10,228 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293970228 2024-11-22T16:44:10,228 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:10,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:10,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T16:44:10,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution in 3.7870 sec 2024-11-22T16:44:10,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:10,234 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:10,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:10,479 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 18 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,479 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:10,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:10,479 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:10,479 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:10,479 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:10,479 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:10,480 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:10,480 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:10,481 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:10,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T16:44:10,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T16:44:10,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:10,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:10,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:10,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:10,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,485 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,485 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,485 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,487 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:44:10,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,488 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:10,488 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:10,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:10,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:10,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:10,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:10,494 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293880494 2024-11-22T16:44:10,495 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:10,495 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,499 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:10,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293910499 2024-11-22T16:44:10,500 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293970500 2024-11-22T16:44:10,500 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:10,500 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:10,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293850501"}]},"ts":"1732293850501"} 2024-11-22T16:44:10,504 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta 2024-11-22T16:44:10,504 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED 2024-11-22T16:44:10,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:10,504 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:10,505 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:10,530 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:44:10,750 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 19 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,750 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:10,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:10,750 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:10,750 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:10,750 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:10,750 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:10,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:10,751 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:10,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:10,753 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:10,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T16:44:10,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T16:44:10,754 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:10,754 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:10,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:10,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:10,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:10,756 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,757 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,757 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:10,758 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T16:44:10,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,759 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:10,760 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:10,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:10,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:10,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:10,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:10,763 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293880763 2024-11-22T16:44:10,763 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:10,763 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:10,767 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:10,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293910767 2024-11-22T16:44:10,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293970767 2024-11-22T16:44:10,767 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T16:44:10,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732293850767"}]},"ts":"1732293850767"} 2024-11-22T16:44:10,770 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta 2024-11-22T16:44:10,770 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED 2024-11-22T16:44:10,770 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:10,772 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T16:44:10,772 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:10,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:10,772 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:11,018 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 20 exec state=pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,018 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T16:44:11,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:11,018 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:11,018 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:11,018 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T16:44:11,018 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T16:44:11,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T16:44:11,019 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:44:11,020 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22, 23] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T16:44:11,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:44:11,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T16:44:11,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T16:44:11,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T16:44:11,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:11,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:11,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T16:44:11,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,024 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1e431292ce15de70c4d2886b2a2b62d3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=08d895a44bedb76f7caed87c8c9bdcf0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=afb9e4c8ad52d77a236d6d40b38f107b, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=42d7a960e5facf87ab193a07a29414cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e431292ce15de70c4d2886b2a2b62d3, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 08d895a44bedb76f7caed87c8c9bdcf0, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure afb9e4c8ad52d77a236d6d40b38f107b, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 42d7a960e5facf87ab193a07a29414cc, server=4618c6346924,46439,1732293836577 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T16:44:11,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T16:44:11,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:11,027 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=4618c6346924,46439,1732293836577, table=hbase:meta, region=1588230740 2024-11-22T16:44:11,027 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,46439,1732293836577, state=OPEN 2024-11-22T16:44:11,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:11,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:44:11,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:11,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:44:11,039 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732293881039 2024-11-22T16:44:11,039 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:44:11,039 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e431292ce15de70c4d2886b2a2b62d3, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:11,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=08d895a44bedb76f7caed87c8c9bdcf0, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:11,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afb9e4c8ad52d77a236d6d40b38f107b, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:11,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42d7a960e5facf87ab193a07a29414cc, regionState=CLOSED, lastHost=4618c6346924,46439,1732293836577, regionLocation=4618c6346924,46439,1732293836577, openSeqNum=2 2024-11-22T16:44:11,044 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:44:11,044 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732293911044 2024-11-22T16:44:11,044 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732293971044 2024-11-22T16:44:11,044 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T16:44:11,045 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T16:44:11,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution in 5.6970 sec 2024-11-22T16:44:11,295 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T16:44:11,297 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testRecoveryAndDoubleExecution 2024-11-22T16:44:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T16:44:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] ipc.CallRunner(138): callId: 13 service: MasterService methodName: DisableTable size: 91 connection: 172.17.0.2:45210 deadline: 1732293911298, exception=org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution 2024-11-22T16:44:11,299 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testRecoveryAndDoubleExecution failed with org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T16:44:11,299 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testRecoveryAndDoubleExecution already disabled, so just deleting it. 2024-11-22T16:44:11,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testRecoveryAndDoubleExecution 2024-11-22T16:44:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,302 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T16:44:11,303 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,310 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:11,310 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:11,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:11,312 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:11,314 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/f1, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/f2, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/recovered.edits] 2024-11-22T16:44:11,316 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/f1, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/f2, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/recovered.edits] 2024-11-22T16:44:11,316 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/f1, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/f2, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/recovered.edits] 2024-11-22T16:44:11,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/f1, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/f2, FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/recovered.edits] 2024-11-22T16:44:11,327 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/recovered.edits/4.seqid to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/archive/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3/recovered.edits/4.seqid 2024-11-22T16:44:11,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/1e431292ce15de70c4d2886b2a2b62d3 2024-11-22T16:44:11,329 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/recovered.edits/4.seqid to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/archive/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b/recovered.edits/4.seqid 2024-11-22T16:44:11,329 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/recovered.edits/4.seqid to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/archive/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0/recovered.edits/4.seqid 2024-11-22T16:44:11,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/recovered.edits/4.seqid to hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/archive/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc/recovered.edits/4.seqid 2024-11-22T16:44:11,330 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/afb9e4c8ad52d77a236d6d40b38f107b 2024-11-22T16:44:11,330 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/08d895a44bedb76f7caed87c8c9bdcf0 2024-11-22T16:44:11,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/default/testRecoveryAndDoubleExecution/42d7a960e5facf87ab193a07a29414cc 2024-11-22T16:44:11,330 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testRecoveryAndDoubleExecution regions 2024-11-22T16:44:11,335 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,338 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 4 rows of testRecoveryAndDoubleExecution from hbase:meta 2024-11-22T16:44:11,342 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testRecoveryAndDoubleExecution' descriptor. 2024-11-22T16:44:11,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testRecoveryAndDoubleExecution' from region states. 2024-11-22T16:44:11,345 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732293851344"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:11,345 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732293851344"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:11,345 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732293851344"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:11,345 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732293851344"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:11,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 4 regions from META 2024-11-22T16:44:11,349 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1e431292ce15de70c4d2886b2a2b62d3, NAME => 'testRecoveryAndDoubleExecution,,1732293843845.1e431292ce15de70c4d2886b2a2b62d3.', STARTKEY => '', ENDKEY => 'a'}, {ENCODED => 08d895a44bedb76f7caed87c8c9bdcf0, NAME => 'testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0.', STARTKEY => 'a', ENDKEY => 'b'}, {ENCODED => afb9e4c8ad52d77a236d6d40b38f107b, NAME => 'testRecoveryAndDoubleExecution,b,1732293843845.afb9e4c8ad52d77a236d6d40b38f107b.', STARTKEY => 'b', ENDKEY => 'c'}, {ENCODED => 42d7a960e5facf87ab193a07a29414cc, NAME => 'testRecoveryAndDoubleExecution,c,1732293843845.42d7a960e5facf87ab193a07a29414cc.', STARTKEY => 'c', ENDKEY => ''}] 2024-11-22T16:44:11,349 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testRecoveryAndDoubleExecution' as deleted. 2024-11-22T16:44:11,349 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732293851349"}]},"ts":"9223372036854775807"} 2024-11-22T16:44:11,352 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testRecoveryAndDoubleExecution state from META 2024-11-22T16:44:11,354 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T16:44:11,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution in 54 msec 2024-11-22T16:44:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T16:44:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testRecoveryAndDoubleExecution 2024-11-22T16:44:11,411 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testRecoveryAndDoubleExecution completed 2024-11-22T16:44:11,420 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=219 (was 224), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=299 (was 289) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8442 (was 8171) - AvailableMemoryMB LEAK? - 2024-11-22T16:44:11,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T16:44:11,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:44:11,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:44:11,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:44:11,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:44:11,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T16:44:11,427 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T16:44:11,427 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=61195404, stopped=false 2024-11-22T16:44:11,427 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4618c6346924,45935,1732293835695 2024-11-22T16:44:11,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:44:11,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:44:11,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:44:11,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:44:11,431 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:11,431 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:44:11,431 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:44:11,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:44:11,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:44:11,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:44:11,431 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,46439,1732293836577' ***** 2024-11-22T16:44:11,432 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T16:44:11,432 INFO [RS:0;4618c6346924:46439 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:44:11,432 INFO [RS:0;4618c6346924:46439 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:44:11,432 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:44:11,432 INFO [RS:0;4618c6346924:46439 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,46439,1732293836577 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4618c6346924:46439. 2024-11-22T16:44:11,433 DEBUG [RS:0;4618c6346924:46439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:44:11,433 DEBUG [RS:0;4618c6346924:46439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:44:11,433 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T16:44:11,434 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T16:44:11,434 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T16:44:11,434 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:44:11,434 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:44:11,434 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:44:11,434 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:44:11,434 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:44:11,434 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T16:44:11,436 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:44:11,438 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=25.04 KB heapSize=40.99 KB 2024-11-22T16:44:11,510 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/info/7917823f0ed742ee9ad8c0530ab532f8 is 108, key is testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0./info:/1732293851344/DeleteFamily/seqid=0 2024-11-22T16:44:11,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741844_1020 (size=6669) 2024-11-22T16:44:11,635 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T16:44:11,835 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T16:44:11,921 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.21 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/info/7917823f0ed742ee9ad8c0530ab532f8 2024-11-22T16:44:11,996 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/ns/7358ee6eb5514311a6c7c6bef23c117a is 106, key is testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0./ns:/1732293851335/DeleteFamily/seqid=0 2024-11-22T16:44:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741845_1021 (size=6009) 2024-11-22T16:44:12,035 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T16:44:12,236 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T16:44:12,394 INFO [regionserver/4618c6346924:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T16:44:12,394 INFO [regionserver/4618c6346924:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T16:44:12,404 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/ns/7358ee6eb5514311a6c7c6bef23c117a 2024-11-22T16:44:12,436 DEBUG [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T16:44:12,448 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/rep_barrier/90c0e4b3ca674347873a025baeb20693 is 115, key is testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0./rep_barrier:/1732293851335/DeleteFamily/seqid=0 2024-11-22T16:44:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741846_1022 (size=6121) 2024-11-22T16:44:12,465 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=648 B at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/rep_barrier/90c0e4b3ca674347873a025baeb20693 2024-11-22T16:44:12,504 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/table/5664998c1b7e40d39c4f171caeb6837b is 109, key is testRecoveryAndDoubleExecution,a,1732293843845.08d895a44bedb76f7caed87c8c9bdcf0./table:/1732293851335/DeleteFamily/seqid=0 2024-11-22T16:44:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741847_1023 (size=6158) 2024-11-22T16:44:12,529 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.55 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/table/5664998c1b7e40d39c4f171caeb6837b 2024-11-22T16:44:12,540 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/info/7917823f0ed742ee9ad8c0530ab532f8 as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/info/7917823f0ed742ee9ad8c0530ab532f8 2024-11-22T16:44:12,549 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/info/7917823f0ed742ee9ad8c0530ab532f8, entries=12, sequenceid=77, filesize=6.5 K 2024-11-22T16:44:12,552 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/ns/7358ee6eb5514311a6c7c6bef23c117a as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/ns/7358ee6eb5514311a6c7c6bef23c117a 2024-11-22T16:44:12,563 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/ns/7358ee6eb5514311a6c7c6bef23c117a, entries=8, sequenceid=77, filesize=5.9 K 2024-11-22T16:44:12,567 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/rep_barrier/90c0e4b3ca674347873a025baeb20693 as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/rep_barrier/90c0e4b3ca674347873a025baeb20693 2024-11-22T16:44:12,582 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/rep_barrier/90c0e4b3ca674347873a025baeb20693, entries=6, sequenceid=77, filesize=6.0 K 2024-11-22T16:44:12,583 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/.tmp/table/5664998c1b7e40d39c4f171caeb6837b as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/table/5664998c1b7e40d39c4f171caeb6837b 2024-11-22T16:44:12,593 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/table/5664998c1b7e40d39c4f171caeb6837b, entries=9, sequenceid=77, filesize=6.0 K 2024-11-22T16:44:12,601 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~25.04 KB/25645, heapSize ~40.93 KB/41912, currentSize=0 B/0 for 1588230740 in 1159ms, sequenceid=77, compaction requested=false 2024-11-22T16:44:12,622 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/data/hbase/meta/1588230740/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1 2024-11-22T16:44:12,622 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:44:12,623 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:44:12,623 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732293851434Running coprocessor pre-close hooks at 1732293851434Disabling compacts and flushes for region at 1732293851434Disabling writes for close at 1732293851434Obtaining lock to block concurrent updates at 1732293851438 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1732293851438Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=25645, getHeapSize=41912, getOffHeapSize=0, getCellsCount=202 at 1732293851449 (+11 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732293851450 (+1 ms)Flushing 1588230740/info: creating writer at 1732293851454 (+4 ms)Flushing 1588230740/info: appending metadata at 1732293851500 (+46 ms)Flushing 1588230740/info: closing flushed file at 1732293851502 (+2 ms)Flushing 1588230740/ns: creating writer at 1732293851974 (+472 ms)Flushing 1588230740/ns: appending metadata at 1732293851994 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732293851994Flushing 1588230740/rep_barrier: creating writer at 1732293852429 (+435 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732293852448 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732293852448Flushing 1588230740/table: creating writer at 1732293852478 (+30 ms)Flushing 1588230740/table: appending metadata at 1732293852503 (+25 ms)Flushing 1588230740/table: closing flushed file at 1732293852503Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119fa53c: reopening flushed file at 1732293852539 (+36 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@383b4cb1: reopening flushed file at 1732293852551 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c87f0fa: reopening flushed file at 1732293852563 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@256772be: reopening flushed file at 1732293852582 (+19 ms)Finished flush of dataSize ~25.04 KB/25645, heapSize ~40.93 KB/41912, currentSize=0 B/0 for 1588230740 in 1159ms, sequenceid=77, compaction requested=false at 1732293852601 (+19 ms)Writing region close event to WAL at 1732293852617 (+16 ms)Running coprocessor post-close hooks at 1732293852622 (+5 ms)Closed at 1732293852623 (+1 ms) 2024-11-22T16:44:12,623 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T16:44:12,636 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,46439,1732293836577; all regions closed. 2024-11-22T16:44:12,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741834_1010 (size=30181) 2024-11-22T16:44:12,645 DEBUG [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/oldWALs 2024-11-22T16:44:12,645 INFO [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C46439%2C1732293836577.meta:.meta(num 1732293839507) 2024-11-22T16:44:12,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741832_1008 (size=4131) 2024-11-22T16:44:12,651 DEBUG [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/oldWALs 2024-11-22T16:44:12,651 INFO [RS:0;4618c6346924:46439 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C46439%2C1732293836577:(num 1732293838488) 2024-11-22T16:44:12,651 DEBUG [RS:0;4618c6346924:46439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:44:12,651 INFO [RS:0;4618c6346924:46439 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:44:12,651 INFO [RS:0;4618c6346924:46439 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:44:12,651 INFO [RS:0;4618c6346924:46439 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T16:44:12,651 INFO [RS:0;4618c6346924:46439 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:44:12,651 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:44:12,652 INFO [RS:0;4618c6346924:46439 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46439 2024-11-22T16:44:12,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:44:12,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,46439,1732293836577 2024-11-22T16:44:12,656 INFO [RS:0;4618c6346924:46439 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:44:12,657 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,46439,1732293836577] 2024-11-22T16:44:12,660 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,46439,1732293836577 already deleted, retry=false 2024-11-22T16:44:12,660 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4618c6346924,46439,1732293836577 expired; onlineServers=0 2024-11-22T16:44:12,660 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4618c6346924,45935,1732293835695' ***** 2024-11-22T16:44:12,660 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T16:44:12,660 INFO [M:0;4618c6346924:45935 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:44:12,661 INFO [M:0;4618c6346924:45935 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:44:12,661 DEBUG [M:0;4618c6346924:45935 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T16:44:12,661 DEBUG [M:0;4618c6346924:45935 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T16:44:12,661 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T16:44:12,661 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293838164 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732293838164,5,FailOnTimeoutGroup] 2024-11-22T16:44:12,661 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293838169 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732293838169,5,FailOnTimeoutGroup] 2024-11-22T16:44:12,661 INFO [M:0;4618c6346924:45935 {}] hbase.ChoreService(370): Chore service for: master/4618c6346924:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T16:44:12,661 INFO [M:0;4618c6346924:45935 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:44:12,661 DEBUG [M:0;4618c6346924:45935 {}] master.HMaster(1795): Stopping service threads 2024-11-22T16:44:12,661 INFO [M:0;4618c6346924:45935 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T16:44:12,662 INFO [M:0;4618c6346924:45935 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:44:12,662 ERROR [M:0;4618c6346924:45935 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-3,5,PEWorkerGroup] Thread[HFileArchiver-4,5,PEWorkerGroup] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T16:44:12,662 INFO [M:0;4618c6346924:45935 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:44:12,662 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T16:44:12,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T16:44:12,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:44:12,664 DEBUG [M:0;4618c6346924:45935 {}] zookeeper.ZKUtil(347): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T16:44:12,664 WARN [M:0;4618c6346924:45935 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T16:44:12,665 INFO [M:0;4618c6346924:45935 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/.lastflushedseqids 2024-11-22T16:44:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741848_1024 (size=72) 2024-11-22T16:44:12,687 INFO [M:0;4618c6346924:45935 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:44:12,687 INFO [M:0;4618c6346924:45935 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T16:44:12,687 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:44:12,687 INFO [M:0;4618c6346924:45935 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:44:12,687 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:44:12,687 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:44:12,687 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:44:12,687 INFO [M:0;4618c6346924:45935 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=149.08 KB heapSize=178.85 KB 2024-11-22T16:44:12,716 DEBUG [M:0;4618c6346924:45935 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f078132c2373491a836b000ffb706bdb is 82, key is hbase:meta,,1/info:regioninfo/1732293839634/Put/seqid=0 2024-11-22T16:44:12,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741849_1025 (size=5672) 2024-11-22T16:44:12,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:44:12,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46439-0x101409022b50001, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:44:12,759 INFO [RS:0;4618c6346924:46439 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:44:12,759 INFO [RS:0;4618c6346924:46439 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,46439,1732293836577; zookeeper connection closed. 2024-11-22T16:44:12,759 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7809816 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7809816 2024-11-22T16:44:12,760 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T16:44:13,126 INFO [M:0;4618c6346924:45935 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f078132c2373491a836b000ffb706bdb 2024-11-22T16:44:13,159 DEBUG [M:0;4618c6346924:45935 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/34fa9ec839454755abe9f0b8a24c7e86 is 1206, key is \x00\x00\x00\x00\x00\x00\x00\x16/proc:d/1732293845180/Put/seqid=0 2024-11-22T16:44:13,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741850_1026 (size=11516) 2024-11-22T16:44:13,168 INFO [M:0;4618c6346924:45935 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148.53 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/34fa9ec839454755abe9f0b8a24c7e86 2024-11-22T16:44:13,205 DEBUG [M:0;4618c6346924:45935 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6c8e5bfbda548ae8bf054edb17736a2 is 69, key is 4618c6346924,46439,1732293836577/rs:state/1732293838223/Put/seqid=0 2024-11-22T16:44:13,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741851_1027 (size=5156) 2024-11-22T16:44:13,620 INFO [M:0;4618c6346924:45935 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6c8e5bfbda548ae8bf054edb17736a2 2024-11-22T16:44:13,628 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f078132c2373491a836b000ffb706bdb as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f078132c2373491a836b000ffb706bdb 2024-11-22T16:44:13,634 INFO [M:0;4618c6346924:45935 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f078132c2373491a836b000ffb706bdb, entries=8, sequenceid=328, filesize=5.5 K 2024-11-22T16:44:13,635 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/34fa9ec839454755abe9f0b8a24c7e86 as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/34fa9ec839454755abe9f0b8a24c7e86 2024-11-22T16:44:13,642 INFO [M:0;4618c6346924:45935 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/34fa9ec839454755abe9f0b8a24c7e86, entries=39, sequenceid=328, filesize=11.2 K 2024-11-22T16:44:13,643 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6c8e5bfbda548ae8bf054edb17736a2 as hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d6c8e5bfbda548ae8bf054edb17736a2 2024-11-22T16:44:13,649 INFO [M:0;4618c6346924:45935 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43711/user/jenkins/test-data/613f4f12-e054-4b4b-6ce1-e9967ef8b4f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d6c8e5bfbda548ae8bf054edb17736a2, entries=1, sequenceid=328, filesize=5.0 K 2024-11-22T16:44:13,650 INFO [M:0;4618c6346924:45935 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.08 KB/152662, heapSize ~178.55 KB/182840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 963ms, sequenceid=328, compaction requested=false 2024-11-22T16:44:13,652 INFO [M:0;4618c6346924:45935 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:44:13,652 DEBUG [M:0;4618c6346924:45935 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732293852687Disabling compacts and flushes for region at 1732293852687Disabling writes for close at 1732293852687Obtaining lock to block concurrent updates at 1732293852687Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732293852687Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=152662, getHeapSize=183080, getOffHeapSize=0, getCellsCount=389 at 1732293852688 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732293852692 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732293852692Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732293852715 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732293852715Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732293853134 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732293853159 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732293853159Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732293853178 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732293853204 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732293853204Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@473a017: reopening flushed file at 1732293853627 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1acb4c28: reopening flushed file at 1732293853634 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d36317: reopening flushed file at 1732293853642 (+8 ms)Finished flush of dataSize ~149.08 KB/152662, heapSize ~178.55 KB/182840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 963ms, sequenceid=328, compaction requested=false at 1732293853650 (+8 ms)Writing region close event to WAL at 1732293853652 (+2 ms)Closed at 1732293853652 2024-11-22T16:44:13,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45621 is added to blk_1073741830_1006 (size=174645) 2024-11-22T16:44:13,655 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:44:13,656 INFO [M:0;4618c6346924:45935 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T16:44:13,656 INFO [M:0;4618c6346924:45935 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45935 2024-11-22T16:44:13,656 INFO [M:0;4618c6346924:45935 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:44:13,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:44:13,759 INFO [M:0;4618c6346924:45935 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:44:13,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45935-0x101409022b50000, quorum=127.0.0.1:59446, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:44:13,764 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@156ffeaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T16:44:13,767 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T16:44:13,767 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T16:44:13,767 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T16:44:13,767 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/hadoop.log.dir/,STOPPED} 2024-11-22T16:44:13,770 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T16:44:13,770 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T16:44:13,770 WARN [BP-887821746-172.17.0.2-1732293832654 heartbeating to localhost/127.0.0.1:43711 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T16:44:13,770 WARN [BP-887821746-172.17.0.2-1732293832654 heartbeating to localhost/127.0.0.1:43711 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-887821746-172.17.0.2-1732293832654 (Datanode Uuid d2b690e1-a7e7-43f3-b22b-ba19fa955d8c) service to localhost/127.0.0.1:43711 2024-11-22T16:44:13,772 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/data/data2/current/BP-887821746-172.17.0.2-1732293832654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T16:44:13,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/cluster_31fc10d4-a78f-5df9-e382-bb169cc9c7a3/data/data1/current/BP-887821746-172.17.0.2-1732293832654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T16:44:13,773 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T16:44:13,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27b19a13{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T16:44:13,785 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T16:44:13,785 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T16:44:13,786 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T16:44:13,786 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1144fac6-5145-ce66-8949-9d070ca859a5/hadoop.log.dir/,STOPPED} 2024-11-22T16:44:13,793 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T16:44:13,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down