2024-11-16 09:43:37,656 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4
2024-11-16 09:43:37,668 main DEBUG Took 0.010132 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-16 09:43:37,669 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-16 09:43:37,669 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-16 09:43:37,670 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-16 09:43:37,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,678 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-16 09:43:37,690 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,692 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,693 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,694 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,695 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,696 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,696 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,697 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,698 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,698 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,699 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,700 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,701 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 09:43:37,702 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,702 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-16 09:43:37,704 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 09:43:37,706 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-16 09:43:37,708 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-16 09:43:37,709 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-16 09:43:37,710 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-16 09:43:37,711 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-16 09:43:37,723 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-16 09:43:37,727 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-16 09:43:37,729 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-16 09:43:37,730 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-16 09:43:37,730 main DEBUG createAppenders(={Console})
2024-11-16 09:43:37,731 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 initialized
2024-11-16 09:43:37,732 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4
2024-11-16 09:43:37,732 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 OK.
2024-11-16 09:43:37,733 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-16 09:43:37,733 main DEBUG OutputStream closed
2024-11-16 09:43:37,734 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-16 09:43:37,734 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-16 09:43:37,734 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4803b726 OK
2024-11-16 09:43:37,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-16 09:43:37,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-16 09:43:37,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-16 09:43:37,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-16 09:43:37,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-16 09:43:37,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-16 09:43:37,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-16 09:43:37,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-16 09:43:37,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-16 09:43:37,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-16 09:43:37,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-16 09:43:37,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-16 09:43:37,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-16 09:43:37,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-16 09:43:37,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-16 09:43:37,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-16 09:43:37,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-16 09:43:37,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-16 09:43:37,830 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-16 09:43:37,830 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null
2024-11-16 09:43:37,830 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-16 09:43:37,831 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK.
2024-11-16T09:43:38,153 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5
2024-11-16 09:43:38,155 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-16 09:43:38,156 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-16T09:43:38,165 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure timeout: 13 mins
2024-11-16T09:43:38,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T09:43:38,198 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c, deleteOnExit=true
2024-11-16T09:43:38,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T09:43:38,199 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/test.cache.data in system properties and HBase conf
2024-11-16T09:43:38,199 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T09:43:38,200 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/hadoop.log.dir in system properties and HBase conf
2024-11-16T09:43:38,200 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T09:43:38,201 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T09:43:38,201 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T09:43:38,299 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-16T09:43:38,386 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T09:43:38,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T09:43:38,391 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T09:43:38,392 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T09:43:38,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T09:43:38,393 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T09:43:38,394 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T09:43:38,395 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T09:43:38,395 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T09:43:38,396 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T09:43:38,397 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/nfs.dump.dir in system properties and HBase conf
2024-11-16T09:43:38,397 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/java.io.tmpdir in system properties and HBase conf
2024-11-16T09:43:38,398 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T09:43:38,398 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T09:43:38,399 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T09:43:39,250 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-16T09:43:39,337 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2424ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-16T09:43:39,421 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T09:43:39,488 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T09:43:39,513 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T09:43:39,514 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T09:43:39,515 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T09:43:39,529 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T09:43:39,532 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/hadoop.log.dir/,AVAILABLE}
2024-11-16T09:43:39,533 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T09:43:39,786 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27b19a13{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/java.io.tmpdir/jetty-localhost-34909-hadoop-hdfs-3_4_1-tests_jar-_-any-2859245602727247863/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T09:43:39,797 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:34909}
2024-11-16T09:43:39,798 INFO  [Time-limited test {}] server.Server(415): Started @2886ms
2024-11-16T09:43:40,238 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T09:43:40,244 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T09:43:40,245 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T09:43:40,246 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T09:43:40,246 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T09:43:40,247 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/hadoop.log.dir/,AVAILABLE}
2024-11-16T09:43:40,247 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T09:43:40,369 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@156ffeaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/java.io.tmpdir/jetty-localhost-41633-hadoop-hdfs-3_4_1-tests_jar-_-any-5133817928746553299/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T09:43:40,370 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:41633}
2024-11-16T09:43:40,370 INFO  [Time-limited test {}] server.Server(415): Started @3458ms
2024-11-16T09:43:40,425 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T09:43:40,882 WARN  [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/data/data1/current/BP-1238687576-172.17.0.2-1731750218989/current, will proceed with Du for space computation calculation, 
2024-11-16T09:43:40,882 WARN  [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/data/data2/current/BP-1238687576-172.17.0.2-1731750218989/current, will proceed with Du for space computation calculation, 
2024-11-16T09:43:40,924 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T09:43:40,979 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2aacb59b2b13fc19 with lease ID 0xb825736ed04e206b: Processing first storage report for DS-bc282ffe-d9e1-43f6-9123-7f8572306384 from datanode DatanodeRegistration(127.0.0.1:36503, datanodeUuid=bce16ce0-27c0-4b48-a7a8-5ce93a5521b6, infoPort=35389, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=2089451024;c=1731750218989)
2024-11-16T09:43:40,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2aacb59b2b13fc19 with lease ID 0xb825736ed04e206b: from storage DS-bc282ffe-d9e1-43f6-9123-7f8572306384 node DatanodeRegistration(127.0.0.1:36503, datanodeUuid=bce16ce0-27c0-4b48-a7a8-5ce93a5521b6, infoPort=35389, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=2089451024;c=1731750218989), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T09:43:40,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2aacb59b2b13fc19 with lease ID 0xb825736ed04e206b: Processing first storage report for DS-7f6977af-ce48-4da3-ba3d-aa83f9bfca5c from datanode DatanodeRegistration(127.0.0.1:36503, datanodeUuid=bce16ce0-27c0-4b48-a7a8-5ce93a5521b6, infoPort=35389, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=2089451024;c=1731750218989)
2024-11-16T09:43:40,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2aacb59b2b13fc19 with lease ID 0xb825736ed04e206b: from storage DS-7f6977af-ce48-4da3-ba3d-aa83f9bfca5c node DatanodeRegistration(127.0.0.1:36503, datanodeUuid=bce16ce0-27c0-4b48-a7a8-5ce93a5521b6, infoPort=35389, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=2089451024;c=1731750218989), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T09:43:41,025 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5
2024-11-16T09:43:41,104 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/zookeeper_0, clientPort=53670, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T09:43:41,113 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53670
2024-11-16T09:43:41,129 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:41,133 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:41,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741825_1001 (size=7)
2024-11-16T09:43:41,787 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b with version=8
2024-11-16T09:43:41,788 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/hbase-staging
2024-11-16T09:43:41,883 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-16T09:43:42,156 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/037c58a19ddb:0 server-side Connection retries=45
2024-11-16T09:43:42,167 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T09:43:42,168 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T09:43:42,173 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T09:43:42,173 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T09:43:42,173 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T09:43:42,312 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T09:43:42,376 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-16T09:43:42,385 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-16T09:43:42,389 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T09:43:42,417 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8919 (auto-detected)
2024-11-16T09:43:42,418 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-16T09:43:42,438 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37567
2024-11-16T09:43:42,461 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37567 connecting to ZooKeeper ensemble=127.0.0.1:53670
2024-11-16T09:43:42,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375670x0, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T09:43:42,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37567-0x101b076ca840000 connected
2024-11-16T09:43:42,519 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:42,521 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:42,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T09:43:42,535 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b, hbase.cluster.distributed=false
2024-11-16T09:43:42,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T09:43:42,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37567
2024-11-16T09:43:42,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37567
2024-11-16T09:43:42,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37567
2024-11-16T09:43:42,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37567
2024-11-16T09:43:42,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37567
2024-11-16T09:43:42,684 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45
2024-11-16T09:43:42,686 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T09:43:42,686 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T09:43:42,686 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T09:43:42,686 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T09:43:42,686 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T09:43:42,689 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T09:43:42,691 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T09:43:42,692 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36121
2024-11-16T09:43:42,694 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36121 connecting to ZooKeeper ensemble=127.0.0.1:53670
2024-11-16T09:43:42,695 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:42,699 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:42,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361210x0, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T09:43:42,706 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36121-0x101b076ca840001 connected
2024-11-16T09:43:42,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T09:43:42,710 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T09:43:42,719 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T09:43:42,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T09:43:42,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T09:43:42,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36121
2024-11-16T09:43:42,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36121
2024-11-16T09:43:42,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36121
2024-11-16T09:43:42,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36121
2024-11-16T09:43:42,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36121
2024-11-16T09:43:42,745 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;037c58a19ddb:37567
2024-11-16T09:43:42,746 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/037c58a19ddb,37567,1731750221953
2024-11-16T09:43:42,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T09:43:42,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T09:43:42,754 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/037c58a19ddb,37567,1731750221953
2024-11-16T09:43:42,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T09:43:42,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:42,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:42,776 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T09:43:42,777 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/037c58a19ddb,37567,1731750221953 from backup master directory
2024-11-16T09:43:42,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/037c58a19ddb,37567,1731750221953
2024-11-16T09:43:42,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T09:43:42,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T09:43:42,781 WARN  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T09:43:42,781 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=037c58a19ddb,37567,1731750221953
2024-11-16T09:43:42,783 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-16T09:43:42,784 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-16T09:43:42,839 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/hbase.id] with ID: 91114306-4c1c-468d-9a8f-fa76d24e43bc
2024-11-16T09:43:42,840 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/.tmp/hbase.id
2024-11-16T09:43:42,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741826_1002 (size=42)
2024-11-16T09:43:43,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/.tmp/hbase.id]:[hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/hbase.id]
2024-11-16T09:43:43,296 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T09:43:43,301 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T09:43:43,320 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-16T09:43:43,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:43,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:43,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741827_1003 (size=196)
2024-11-16T09:43:43,755 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T09:43:43,758 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T09:43:43,775 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:43,779 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-16T09:43:43,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741828_1004 (size=1189)
2024-11-16T09:43:44,229 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store
2024-11-16T09:43:44,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741829_1005 (size=34)
2024-11-16T09:43:44,654 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-16T09:43:44,657 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:44,659 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T09:43:44,659 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:44,659 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:44,661 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T09:43:44,661 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:44,661 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:44,663 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731750224658Disabling compacts and flushes for region at 1731750224658Disabling writes for close at 1731750224661 (+3 ms)Writing region close event to WAL at 1731750224661Closed at 1731750224661
2024-11-16T09:43:44,665 WARN  [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/.initializing
2024-11-16T09:43:44,665 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/WALs/037c58a19ddb,37567,1731750221953
2024-11-16T09:43:44,676 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-16T09:43:44,692 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C37567%2C1731750221953, suffix=, logDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/WALs/037c58a19ddb,37567,1731750221953, archiveDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/oldWALs, maxLogs=10
2024-11-16T09:43:44,716 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/WALs/037c58a19ddb,37567,1731750221953/037c58a19ddb%2C37567%2C1731750221953.1731750224696, exclude list is [], retry=0
2024-11-16T09:43:44,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36503,DS-bc282ffe-d9e1-43f6-9123-7f8572306384,DISK]
2024-11-16T09:43:44,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf.
2024-11-16T09:43:44,774 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/WALs/037c58a19ddb,37567,1731750221953/037c58a19ddb%2C37567%2C1731750221953.1731750224696
2024-11-16T09:43:44,774 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35389:35389)]
2024-11-16T09:43:44,775 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T09:43:44,775 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:44,779 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,780 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,818 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T09:43:44,846 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T09:43:44,850 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:44,853 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:44,854 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T09:43:44,857 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T09:43:44,857 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:44,858 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:44,859 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T09:43:44,862 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T09:43:44,862 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:44,863 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:44,863 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T09:43:44,866 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T09:43:44,866 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:44,867 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:44,867 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,870 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,871 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,876 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,877 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,880 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T09:43:44,883 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T09:43:44,887 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:44,888 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70653819, jitterRate=0.05282394587993622}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T09:43:44,894 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731750224791Initializing all the Stores at 1731750224794 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750224794Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750224795 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750224796 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750224796Cleaning up temporary data from old regions at 1731750224877 (+81 ms)Region opened successfully at 1731750224894 (+17 ms)
2024-11-16T09:43:44,895 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T09:43:44,928 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17908b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0
2024-11-16T09:43:44,961 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T09:43:44,972 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:44,972 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:44,975 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:44,977 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:44,982 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:44,982 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T09:43:45,007 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T09:43:45,014 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T09:43:45,016 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T09:43:45,019 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T09:43:45,020 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T09:43:45,021 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T09:43:45,023 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T09:43:45,027 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T09:43:45,028 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T09:43:45,029 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T09:43:45,031 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T09:43:45,048 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T09:43:45,050 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T09:43:45,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T09:43:45,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T09:43:45,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:45,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:45,056 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=037c58a19ddb,37567,1731750221953, sessionid=0x101b076ca840000, setting cluster-up flag (Was=false)
2024-11-16T09:43:45,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:45,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:45,075 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T09:43:45,077 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,37567,1731750221953
2024-11-16T09:43:45,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:45,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:45,087 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T09:43:45,088 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,37567,1731750221953
2024-11-16T09:43:45,095 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T09:43:45,135 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(746): ClusterId : 91114306-4c1c-468d-9a8f-fa76d24e43bc
2024-11-16T09:43:45,138 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T09:43:45,143 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T09:43:45,143 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T09:43:45,146 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T09:43:45,146 DEBUG [RS:0;037c58a19ddb:36121 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38748a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0
2024-11-16T09:43:45,162 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;037c58a19ddb:36121
2024-11-16T09:43:45,166 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T09:43:45,166 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T09:43:45,166 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T09:43:45,168 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,37567,1731750221953 with port=36121, startcode=1731750222645
2024-11-16T09:43:45,180 DEBUG [RS:0;037c58a19ddb:36121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T09:43:45,181 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:45,192 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T09:43:45,202 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T09:43:45,210 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 037c58a19ddb,37567,1731750221953
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T09:43:45,221 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5
2024-11-16T09:43:45,221 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42165, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T09:43:45,221 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5
2024-11-16T09:43:45,221 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5
2024-11-16T09:43:45,222 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5
2024-11-16T09:43:45,222 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/037c58a19ddb:0, corePoolSize=10, maxPoolSize=10
2024-11-16T09:43:45,222 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,223 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2
2024-11-16T09:43:45,223 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,224 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750255224
2024-11-16T09:43:45,226 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T09:43:45,227 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T09:43:45,228 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:45,228 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T09:43:45,228 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:45,232 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T09:43:45,232 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T09:43:45,233 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T09:43:45,233 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T09:43:45,234 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:45,234 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T09:43:45,234 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,240 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T09:43:45,241 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T09:43:45,242 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T09:43:45,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741831_1007 (size=1321)
2024-11-16T09:43:45,245 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T09:43:45,245 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T09:43:45,248 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750225246,5,FailOnTimeoutGroup]
2024-11-16T09:43:45,248 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750225248,5,FailOnTimeoutGroup]
2024-11-16T09:43:45,249 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,249 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T09:43:45,250 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,250 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,257 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-11-16T09:43:45,257 WARN  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-11-16T09:43:45,358 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,37567,1731750221953 with port=36121, startcode=1731750222645
2024-11-16T09:43:45,360 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,36121,1731750222645
2024-11-16T09:43:45,363 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:45,372 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:45,372 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32973
2024-11-16T09:43:45,372 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T09:43:45,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T09:43:45,376 DEBUG [RS:0;037c58a19ddb:36121 {}] zookeeper.ZKUtil(111): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,36121,1731750222645
2024-11-16T09:43:45,377 WARN  [RS:0;037c58a19ddb:36121 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T09:43:45,377 INFO  [RS:0;037c58a19ddb:36121 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-16T09:43:45,377 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645
2024-11-16T09:43:45,379 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,36121,1731750222645]
2024-11-16T09:43:45,402 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T09:43:45,416 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T09:43:45,421 INFO  [RS:0;037c58a19ddb:36121 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T09:43:45,421 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,422 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T09:43:45,427 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T09:43:45,428 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,429 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,429 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,429 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,429 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,430 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,430 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2
2024-11-16T09:43:45,430 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,430 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,431 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,431 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,431 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,431 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1
2024-11-16T09:43:45,432 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3
2024-11-16T09:43:45,432 DEBUG [RS:0;037c58a19ddb:36121 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3
2024-11-16T09:43:45,433 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,433 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,433 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,433 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,434 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,434 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,36121,1731750222645-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T09:43:45,454 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T09:43:45,456 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,36121,1731750222645-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,456 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,457 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.Replication(171): 037c58a19ddb,36121,1731750222645 started
2024-11-16T09:43:45,475 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:45,476 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,36121,1731750222645, RpcServer on 037c58a19ddb/172.17.0.2:36121, sessionid=0x101b076ca840001
2024-11-16T09:43:45,476 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T09:43:45,477 DEBUG [RS:0;037c58a19ddb:36121 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,36121,1731750222645
2024-11-16T09:43:45,477 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,36121,1731750222645'
2024-11-16T09:43:45,477 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T09:43:45,478 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T09:43:45,479 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T09:43:45,479 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T09:43:45,479 DEBUG [RS:0;037c58a19ddb:36121 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,36121,1731750222645
2024-11-16T09:43:45,479 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,36121,1731750222645'
2024-11-16T09:43:45,479 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T09:43:45,480 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T09:43:45,480 DEBUG [RS:0;037c58a19ddb:36121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T09:43:45,480 INFO  [RS:0;037c58a19ddb:36121 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T09:43:45,481 INFO  [RS:0;037c58a19ddb:36121 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T09:43:45,586 INFO  [RS:0;037c58a19ddb:36121 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-16T09:43:45,589 INFO  [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C36121%2C1731750222645, suffix=, logDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645, archiveDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/oldWALs, maxLogs=32
2024-11-16T09:43:45,607 DEBUG [RS:0;037c58a19ddb:36121 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645/037c58a19ddb%2C36121%2C1731750222645.1731750225592, exclude list is [], retry=0
2024-11-16T09:43:45,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36503,DS-bc282ffe-d9e1-43f6-9123-7f8572306384,DISK]
2024-11-16T09:43:45,615 INFO  [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645/037c58a19ddb%2C36121%2C1731750222645.1731750225592
2024-11-16T09:43:45,615 DEBUG [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35389:35389)]
2024-11-16T09:43:45,646 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T09:43:45,646 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:45,655 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741833_1009 (size=32)
2024-11-16T09:43:46,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:46,060 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T09:43:46,063 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T09:43:46,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,064 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,065 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T09:43:46,067 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T09:43:46,067 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,068 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,068 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T09:43:46,070 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T09:43:46,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,071 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,071 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T09:43:46,074 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T09:43:46,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,075 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T09:43:46,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740
2024-11-16T09:43:46,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740
2024-11-16T09:43:46,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T09:43:46,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T09:43:46,081 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T09:43:46,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T09:43:46,087 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:46,087 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61783348, jitterRate=-0.0793563723564148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T09:43:46,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731750226058Initializing all the Stores at 1731750226060 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750226060Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750226060Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750226060Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750226060Cleaning up temporary data from old regions at 1731750226080 (+20 ms)Region opened successfully at 1731750226089 (+9 ms)
2024-11-16T09:43:46,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T09:43:46,090 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T09:43:46,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T09:43:46,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T09:43:46,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T09:43:46,091 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T09:43:46,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731750226090Disabling compacts and flushes for region at 1731750226090Disabling writes for close at 1731750226090Writing region close event to WAL at 1731750226091 (+1 ms)Closed at 1731750226091
2024-11-16T09:43:46,094 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:46,095 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T09:43:46,101 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T09:43:46,111 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T09:43:46,113 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T09:43:46,266 DEBUG [037c58a19ddb:37567 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T09:43:46,278 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:46,286 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPENING
2024-11-16T09:43:46,291 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T09:43:46,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:46,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:46,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:46,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:46,295 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T09:43:46,297 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:46,472 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T09:43:46,476 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T09:43:46,487 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T09:43:46,487 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-16T09:43:46,488 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-11-16T09:43:46,491 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C36121%2C1731750222645.meta, suffix=.meta, logDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645, archiveDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/oldWALs, maxLogs=32
2024-11-16T09:43:46,508 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645/037c58a19ddb%2C36121%2C1731750222645.meta.1731750226493.meta, exclude list is [], retry=0
2024-11-16T09:43:46,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36503,DS-bc282ffe-d9e1-43f6-9123-7f8572306384,DISK]
2024-11-16T09:43:46,514 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/WALs/037c58a19ddb,36121,1731750222645/037c58a19ddb%2C36121%2C1731750222645.meta.1731750226493.meta
2024-11-16T09:43:46,515 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35389:35389)]
2024-11-16T09:43:46,515 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T09:43:46,518 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T09:43:46,521 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T09:43:46,527 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T09:43:46,533 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T09:43:46,534 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:46,534 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T09:43:46,534 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T09:43:46,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T09:43:46,540 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T09:43:46,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,541 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,541 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T09:43:46,543 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T09:43:46,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,544 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,544 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T09:43:46,545 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T09:43:46,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,546 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,546 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T09:43:46,548 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T09:43:46,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,549 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T09:43:46,549 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T09:43:46,551 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740
2024-11-16T09:43:46,554 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740
2024-11-16T09:43:46,556 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T09:43:46,557 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T09:43:46,558 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T09:43:46,560 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T09:43:46,562 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64025624, jitterRate=-0.04594385623931885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T09:43:46,562 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T09:43:46,564 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731750226535Writing region info on filesystem at 1731750226535Initializing all the Stores at 1731750226537 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750226538 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750226538Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750226538Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750226538Cleaning up temporary data from old regions at 1731750226557 (+19 ms)Running coprocessor post-open hooks at 1731750226562 (+5 ms)Region opened successfully at 1731750226564 (+2 ms)
2024-11-16T09:43:46,572 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731750226463
2024-11-16T09:43:46,584 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T09:43:46,584 INFO  [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T09:43:46,586 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:46,588 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:46,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:46,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:46,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:46,592 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:46,593 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:46,597 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T09:43:46,598 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,36121,1731750222645 in 296 msec
2024-11-16T09:43:46,604 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T09:43:46,604 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 498 msec
2024-11-16T09:43:46,605 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:46,605 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T09:43:46,627 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T09:43:46,629 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,36121,1731750222645, seqNum=-1]
2024-11-16T09:43:46,653 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T09:43:46,656 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41383, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T09:43:46,684 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5580 sec
2024-11-16T09:43:46,684 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731750226684, completionTime=-1
2024-11-16T09:43:46,687 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T09:43:46,687 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:46,713 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:46,714 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750286714
2024-11-16T09:43:46,714 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750346714
2024-11-16T09:43:46,714 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec
2024-11-16T09:43:46,716 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37567,1731750221953-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,716 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37567,1731750221953-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,716 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37567,1731750221953-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,718 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-037c58a19ddb:37567, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,718 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,719 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,724 DEBUG [master/037c58a19ddb:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T09:43:46,746 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.964sec
2024-11-16T09:43:46,747 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T09:43:46,748 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T09:43:46,749 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T09:43:46,750 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T09:43:46,750 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T09:43:46,750 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37567,1731750221953-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T09:43:46,751 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37567,1731750221953-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T09:43:46,759 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T09:43:46,760 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T09:43:46,760 INFO  [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,37567,1731750221953-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T09:43:46,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d7c978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T09:43:46,848 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-16T09:43:46,848 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-16T09:43:46,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,37567,-1 for getting cluster id
2024-11-16T09:43:46,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T09:43:46,862 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '91114306-4c1c-468d-9a8f-fa76d24e43bc'
2024-11-16T09:43:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T09:43:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "91114306-4c1c-468d-9a8f-fa76d24e43bc"

2024-11-16T09:43:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbd2bc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T09:43:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,37567,-1]
2024-11-16T09:43:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T09:43:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T09:43:46,870 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T09:43:46,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df26bdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T09:43:46,874 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T09:43:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,36121,1731750222645, seqNum=-1]
2024-11-16T09:43:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T09:43:46,883 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T09:43:46,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=037c58a19ddb,37567,1731750221953
2024-11-16T09:43:46,920 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTable Thread=207, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=6158
2024-11-16T09:43:46,925 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-16T09:43:46,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:46,956 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T09:43:46,960 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:46,962 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T09:43:46,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741835_1011 (size=639)
2024-11-16T09:43:47,379 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 426e2ea06beb440d31bf9f4c8fd9effa, NAME => 'testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:47,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741836_1012 (size=51)
2024-11-16T09:43:47,789 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:47,789 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1722): Closing 426e2ea06beb440d31bf9f4c8fd9effa, disabling compactions & flushes
2024-11-16T09:43:47,789 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:47,789 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:47,789 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa. after waiting 0 ms
2024-11-16T09:43:47,789 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:47,789 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:47,789 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 426e2ea06beb440d31bf9f4c8fd9effa:
Waiting for close lock at 1731750227789Disabling compacts and flushes for region at 1731750227789Disabling writes for close at 1731750227789Writing region close event to WAL at 1731750227789Closed at 1731750227789
2024-11-16T09:43:47,792 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T09:43:47,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731750227792"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750227792"}]},"ts":"1731750227792"}
2024-11-16T09:43:47,802 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T09:43:47,804 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T09:43:47,807 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750227804"}]},"ts":"1731750227804"}
2024-11-16T09:43:47,812 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLING in hbase:meta
2024-11-16T09:43:47,814 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, ASSIGN}]
2024-11-16T09:43:47,817 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, ASSIGN
2024-11-16T09:43:47,818 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, ASSIGN; state=OFFLINE, location=037c58a19ddb,36121,1731750222645; forceNewPlan=false, retain=false
2024-11-16T09:43:47,969 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=426e2ea06beb440d31bf9f4c8fd9effa, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:47,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, ASSIGN because future has completed
2024-11-16T09:43:47,974 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 426e2ea06beb440d31bf9f4c8fd9effa, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:48,134 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,135 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 426e2ea06beb440d31bf9f4c8fd9effa, NAME => 'testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.', STARTKEY => '', ENDKEY => ''}
2024-11-16T09:43:48,135 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTable 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,135 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:48,136 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,136 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,139 INFO  [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 426e2ea06beb440d31bf9f4c8fd9effa 
2024-11-16T09:43:48,141 INFO  [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426e2ea06beb440d31bf9f4c8fd9effa columnFamilyName f1
2024-11-16T09:43:48,141 DEBUG [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:48,142 INFO  [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] regionserver.HStore(327): Store=426e2ea06beb440d31bf9f4c8fd9effa/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:48,142 INFO  [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 426e2ea06beb440d31bf9f4c8fd9effa 
2024-11-16T09:43:48,144 INFO  [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426e2ea06beb440d31bf9f4c8fd9effa columnFamilyName f2
2024-11-16T09:43:48,145 DEBUG [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:48,145 INFO  [StoreOpener-426e2ea06beb440d31bf9f4c8fd9effa-1 {}] regionserver.HStore(327): Store=426e2ea06beb440d31bf9f4c8fd9effa/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:48,146 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,147 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,148 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,149 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,149 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,150 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTable descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-16T09:43:48,152 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,156 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:48,157 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 426e2ea06beb440d31bf9f4c8fd9effa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66788005, jitterRate=-0.004781171679496765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-16T09:43:48,157 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,158 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 426e2ea06beb440d31bf9f4c8fd9effa:
Running coprocessor pre-open hook at 1731750228136Writing region info on filesystem at 1731750228136Initializing all the Stores at 1731750228138 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750228138Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750228138Cleaning up temporary data from old regions at 1731750228149 (+11 ms)Running coprocessor post-open hooks at 1731750228157 (+8 ms)Region opened successfully at 1731750228158 (+1 ms)
2024-11-16T09:43:48,160 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa., pid=6, masterSystemTime=1731750228128
2024-11-16T09:43:48,164 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,164 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,165 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=426e2ea06beb440d31bf9f4c8fd9effa, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:48,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 426e2ea06beb440d31bf9f4c8fd9effa, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:48,175 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-16T09:43:48,175 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 426e2ea06beb440d31bf9f4c8fd9effa, server=037c58a19ddb,36121,1731750222645 in 197 msec
2024-11-16T09:43:48,179 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-16T09:43:48,180 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, ASSIGN in 363 msec
2024-11-16T09:43:48,181 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T09:43:48,182 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750228181"}]},"ts":"1731750228181"}
2024-11-16T09:43:48,185 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLED in hbase:meta
2024-11-16T09:43:48,186 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T09:43:48,190 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable in 1.2580 sec
2024-11-16T09:43:48,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:48,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750228214"}]},"ts":"1731750228214"}
2024-11-16T09:43:48,218 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLING in hbase:meta
2024-11-16T09:43:48,218 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTable to state=DISABLING
2024-11-16T09:43:48,220 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTable}]
2024-11-16T09:43:48,227 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, UNASSIGN}]
2024-11-16T09:43:48,230 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, UNASSIGN
2024-11-16T09:43:48,232 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=426e2ea06beb440d31bf9f4c8fd9effa, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:48,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, UNASSIGN because future has completed
2024-11-16T09:43:48,237 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:48,237 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 426e2ea06beb440d31bf9f4c8fd9effa, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:48,396 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,396 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-16T09:43:48,397 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 426e2ea06beb440d31bf9f4c8fd9effa, disabling compactions & flushes
2024-11-16T09:43:48,397 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,397 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,397 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa. after waiting 0 ms
2024-11-16T09:43:48,397 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,405 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-16T09:43:48,408 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.
2024-11-16T09:43:48,408 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 426e2ea06beb440d31bf9f4c8fd9effa:
Waiting for close lock at 1731750228397Running coprocessor pre-close hooks at 1731750228397Disabling compacts and flushes for region at 1731750228397Disabling writes for close at 1731750228397Writing region close event to WAL at 1731750228398 (+1 ms)Running coprocessor post-close hooks at 1731750228406 (+8 ms)Closed at 1731750228408 (+2 ms)
2024-11-16T09:43:48,412 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,413 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=426e2ea06beb440d31bf9f4c8fd9effa, regionState=CLOSED
2024-11-16T09:43:48,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 426e2ea06beb440d31bf9f4c8fd9effa, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:48,421 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-16T09:43:48,421 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 426e2ea06beb440d31bf9f4c8fd9effa, server=037c58a19ddb,36121,1731750222645 in 181 msec
2024-11-16T09:43:48,425 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-16T09:43:48,425 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=426e2ea06beb440d31bf9f4c8fd9effa, UNASSIGN in 195 msec
2024-11-16T09:43:48,430 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-16T09:43:48,430 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTable in 206 msec
2024-11-16T09:43:48,433 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750228433"}]},"ts":"1731750228433"}
2024-11-16T09:43:48,437 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLED in hbase:meta
2024-11-16T09:43:48,437 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTable to state=DISABLED
2024-11-16T09:43:48,441 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable in 233 msec
2024-11-16T09:43:48,461 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-16T09:43:48,464 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-16T09:43:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,37567,1731750221953
2024-11-16T09:43:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3108930d
2024-11-16T09:43:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-16T09:43:48,475 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-16T09:43:48,478 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTable
2024-11-16T09:43:48,482 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTable
2024-11-16T09:43:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] ipc.CallRunner(138): callId: 2 service: MasterService methodName: DisableTable size: 78 connection: 172.17.0.2:54140 deadline: 1731750288481, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
2024-11-16T09:43:48,487 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTable failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-16T09:43:48,487 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTable already disabled, so just deleting it.
2024-11-16T09:43:48,490 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTable
2024-11-16T09:43:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] procedure2.ProcedureExecutor(1139): Stored pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:48,499 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:48,502 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=12, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12
2024-11-16T09:43:48,511 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,516 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/f1, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/f2, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/recovered.edits]
2024-11-16T09:43:48,531 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/recovered.edits/4.seqid to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/archive/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa/recovered.edits/4.seqid
2024-11-16T09:43:48,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTable/426e2ea06beb440d31bf9f4c8fd9effa
2024-11-16T09:43:48,532 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTable regions
2024-11-16T09:43:48,539 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=12, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:48,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36121 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms
2024-11-16T09:43:48,549 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTable from hbase:meta
2024-11-16T09:43:48,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTable' descriptor.
2024-11-16T09:43:48,588 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=12, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:48,588 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTable' from region states.
2024-11-16T09:43:48,589 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731750228588"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:48,594 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META
2024-11-16T09:43:48,594 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 426e2ea06beb440d31bf9f4c8fd9effa, NAME => 'testDisableTable,,1731750226927.426e2ea06beb440d31bf9f4c8fd9effa.', STARTKEY => '', ENDKEY => ''}]
2024-11-16T09:43:48,594 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTable' as deleted.
2024-11-16T09:43:48,595 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731750228595"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:48,598 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTable state from META
2024-11-16T09:43:48,600 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=12, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:48,602 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable in 108 msec
2024-11-16T09:43:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12
2024-11-16T09:43:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTable
2024-11-16T09:43:48,619 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTable completed
2024-11-16T09:43:48,629 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTable Thread=217 (was 207)
Potentially hanging thread: RSProcedureDispatcher-pool-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_OPEN_REGION-regionserver/037c58a19ddb:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_31153621_22 at /127.0.0.1:58614 [Waiting for operation #6]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 311), ProcessCount=11 (was 11), AvailableMemoryMB=6105 (was 6158)
2024-11-16T09:43:48,638 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=217, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=6103
2024-11-16T09:43:48,638 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-16T09:43:48,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:48,642 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T09:43:48,642 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:48,644 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T09:43:48,653 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741837_1013 (size=652)
2024-11-16T09:43:49,057 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe5e2dce90eb3e4c03d3cf342c88f670, NAME => 'testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTableMultipleTimes', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:49,065 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741838_1014 (size=64)
2024-11-16T09:43:49,466 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:49,466 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1722): Closing fe5e2dce90eb3e4c03d3cf342c88f670, disabling compactions & flushes
2024-11-16T09:43:49,467 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,467 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,467 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670. after waiting 0 ms
2024-11-16T09:43:49,467 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,467 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,467 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe5e2dce90eb3e4c03d3cf342c88f670:
Waiting for close lock at 1731750229466Disabling compacts and flushes for region at 1731750229466Disabling writes for close at 1731750229467 (+1 ms)Writing region close event to WAL at 1731750229467Closed at 1731750229467
2024-11-16T09:43:49,469 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T09:43:49,469 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731750229469"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750229469"}]},"ts":"1731750229469"}
2024-11-16T09:43:49,472 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T09:43:49,473 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T09:43:49,474 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750229473"}]},"ts":"1731750229473"}
2024-11-16T09:43:49,476 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLING in hbase:meta
2024-11-16T09:43:49,477 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, ASSIGN}]
2024-11-16T09:43:49,479 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, ASSIGN
2024-11-16T09:43:49,480 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, ASSIGN; state=OFFLINE, location=037c58a19ddb,36121,1731750222645; forceNewPlan=false, retain=false
2024-11-16T09:43:49,631 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fe5e2dce90eb3e4c03d3cf342c88f670, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:49,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, ASSIGN because future has completed
2024-11-16T09:43:49,636 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe5e2dce90eb3e4c03d3cf342c88f670, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:49,794 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,795 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => fe5e2dce90eb3e4c03d3cf342c88f670, NAME => 'testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.', STARTKEY => '', ENDKEY => ''}
2024-11-16T09:43:49,795 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTableMultipleTimes fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,795 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:49,795 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,795 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,797 INFO  [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region fe5e2dce90eb3e4c03d3cf342c88f670 
2024-11-16T09:43:49,800 INFO  [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe5e2dce90eb3e4c03d3cf342c88f670 columnFamilyName f1
2024-11-16T09:43:49,800 DEBUG [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:49,800 INFO  [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] regionserver.HStore(327): Store=fe5e2dce90eb3e4c03d3cf342c88f670/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:49,801 INFO  [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region fe5e2dce90eb3e4c03d3cf342c88f670 
2024-11-16T09:43:49,804 INFO  [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe5e2dce90eb3e4c03d3cf342c88f670 columnFamilyName f2
2024-11-16T09:43:49,804 DEBUG [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:49,805 INFO  [StoreOpener-fe5e2dce90eb3e4c03d3cf342c88f670-1 {}] regionserver.HStore(327): Store=fe5e2dce90eb3e4c03d3cf342c88f670/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:49,805 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,806 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,807 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,808 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,808 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,809 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTableMultipleTimes descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-16T09:43:49,811 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,814 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:49,815 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened fe5e2dce90eb3e4c03d3cf342c88f670; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58961183, jitterRate=-0.12140990793704987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-16T09:43:49,815 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:49,816 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for fe5e2dce90eb3e4c03d3cf342c88f670:
Running coprocessor pre-open hook at 1731750229796Writing region info on filesystem at 1731750229796Initializing all the Stores at 1731750229797 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750229797Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750229797Cleaning up temporary data from old regions at 1731750229808 (+11 ms)Running coprocessor post-open hooks at 1731750229815 (+7 ms)Region opened successfully at 1731750229816 (+1 ms)
2024-11-16T09:43:49,817 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670., pid=15, masterSystemTime=1731750229789
2024-11-16T09:43:49,820 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,820 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:49,821 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fe5e2dce90eb3e4c03d3cf342c88f670, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:49,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe5e2dce90eb3e4c03d3cf342c88f670, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:49,830 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14
2024-11-16T09:43:49,830 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure fe5e2dce90eb3e4c03d3cf342c88f670, server=037c58a19ddb,36121,1731750222645 in 191 msec
2024-11-16T09:43:49,833 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-16T09:43:49,833 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, ASSIGN in 354 msec
2024-11-16T09:43:49,834 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T09:43:49,834 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750229834"}]},"ts":"1731750229834"}
2024-11-16T09:43:49,837 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLED in hbase:meta
2024-11-16T09:43:49,838 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T09:43:49,840 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes in 1.2000 sec
2024-11-16T09:43:49,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:49,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750229896"}]},"ts":"1731750229896"}
2024-11-16T09:43:49,899 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLING in hbase:meta
2024-11-16T09:43:49,899 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTableMultipleTimes to state=DISABLING
2024-11-16T09:43:49,900 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes}]
2024-11-16T09:43:49,903 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, UNASSIGN}]
2024-11-16T09:43:49,905 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, UNASSIGN
2024-11-16T09:43:49,906 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fe5e2dce90eb3e4c03d3cf342c88f670, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:49,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, UNASSIGN because future has completed
2024-11-16T09:43:49,909 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:49,910 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe5e2dce90eb3e4c03d3cf342c88f670, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:50,063 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(122): Close fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:50,064 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-16T09:43:50,064 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1722): Closing fe5e2dce90eb3e4c03d3cf342c88f670, disabling compactions & flushes
2024-11-16T09:43:50,064 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:50,064 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:50,064 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670. after waiting 0 ms
2024-11-16T09:43:50,064 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:50,071 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-16T09:43:50,072 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.
2024-11-16T09:43:50,072 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1676): Region close journal for fe5e2dce90eb3e4c03d3cf342c88f670:
Waiting for close lock at 1731750230064Running coprocessor pre-close hooks at 1731750230064Disabling compacts and flushes for region at 1731750230064Disabling writes for close at 1731750230064Writing region close event to WAL at 1731750230065 (+1 ms)Running coprocessor post-close hooks at 1731750230071 (+6 ms)Closed at 1731750230072 (+1 ms)
2024-11-16T09:43:50,075 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(157): Closed fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:50,076 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fe5e2dce90eb3e4c03d3cf342c88f670, regionState=CLOSED
2024-11-16T09:43:50,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe5e2dce90eb3e4c03d3cf342c88f670, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:50,083 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18
2024-11-16T09:43:50,083 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; CloseRegionProcedure fe5e2dce90eb3e4c03d3cf342c88f670, server=037c58a19ddb,36121,1731750222645 in 170 msec
2024-11-16T09:43:50,086 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17
2024-11-16T09:43:50,086 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=fe5e2dce90eb3e4c03d3cf342c88f670, UNASSIGN in 180 msec
2024-11-16T09:43:50,091 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16
2024-11-16T09:43:50,091 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes in 187 msec
2024-11-16T09:43:50,093 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750230093"}]},"ts":"1731750230093"}
2024-11-16T09:43:50,096 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLED in hbase:meta
2024-11-16T09:43:50,096 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTableMultipleTimes to state=DISABLED
2024-11-16T09:43:50,100 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes in 206 msec
2024-11-16T09:43:50,143 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(101): Disable failed with exception {}org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
2024-11-16T09:43:50,143 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(114): Disable failed with expected exception {}
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:109) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:50,144 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(126): Disable failed with expected exception {}
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:64) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:120) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:50,145 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-16T09:43:50,149 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTableMultipleTimes
2024-11-16T09:43:50,150 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTableMultipleTimes
2024-11-16T09:43:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] ipc.CallRunner(138): callId: 7 service: MasterService methodName: DisableTable size: 91 connection: 172.17.0.2:54140 deadline: 1731750290150, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
2024-11-16T09:43:50,152 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTableMultipleTimes failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-16T09:43:50,152 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTableMultipleTimes already disabled, so just deleting it.
2024-11-16T09:43:50,155 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTableMultipleTimes
2024-11-16T09:43:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:50,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-16T09:43:50,162 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:50,176 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:50,183 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/f1, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/f2, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/recovered.edits]
2024-11-16T09:43:50,204 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/recovered.edits/4.seqid to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/archive/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670/recovered.edits/4.seqid
2024-11-16T09:43:50,205 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testDisableTableMultipleTimes/fe5e2dce90eb3e4c03d3cf342c88f670
2024-11-16T09:43:50,205 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTableMultipleTimes regions
2024-11-16T09:43:50,212 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:50,217 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTableMultipleTimes from hbase:meta
2024-11-16T09:43:50,221 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTableMultipleTimes' descriptor.
2024-11-16T09:43:50,225 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:50,225 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTableMultipleTimes' from region states.
2024-11-16T09:43:50,225 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731750230225"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:50,230 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META
2024-11-16T09:43:50,230 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe5e2dce90eb3e4c03d3cf342c88f670, NAME => 'testDisableTableMultipleTimes,,1731750228638.fe5e2dce90eb3e4c03d3cf342c88f670.', STARTKEY => '', ENDKEY => ''}]
2024-11-16T09:43:50,230 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTableMultipleTimes' as deleted.
2024-11-16T09:43:50,230 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731750230230"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:50,234 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTableMultipleTimes state from META
2024-11-16T09:43:50,236 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:50,238 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes in 81 msec
2024-11-16T09:43:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-16T09:43:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTableMultipleTimes
2024-11-16T09:43:50,272 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTableMultipleTimes completed
2024-11-16T09:43:50,282 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=224 (was 217)
Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/data/data1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/data/data2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_31153621_22 at /127.0.0.1:58614 [Waiting for operation #7]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=294 (was 311), ProcessCount=11 (was 11), AvailableMemoryMB=6093 (was 6103)
2024-11-16T09:43:50,292 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=224, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=6087
2024-11-16T09:43:50,292 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-16T09:43:50,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:50,297 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T09:43:50,297 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:50,299 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T09:43:50,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741839_1015 (size=653)
2024-11-16T09:43:50,712 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7db3eb8b266db3675f5937e0dbfbe0e9, NAME => 'testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:50,712 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 416cf44c077ffd5b666389825d361040, NAME => 'testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:50,712 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b7087517245861969bfc8373d6d025e3, NAME => 'testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:50,723 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741840_1016 (size=67)
2024-11-16T09:43:50,724 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741842_1018 (size=67)
2024-11-16T09:43:50,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741841_1017 (size=66)
2024-11-16T09:43:50,725 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:50,726 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 416cf44c077ffd5b666389825d361040, disabling compactions & flushes
2024-11-16T09:43:50,726 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:50,726 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:50,726 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040. after waiting 0 ms
2024-11-16T09:43:50,726 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:50,726 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:50,726 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 416cf44c077ffd5b666389825d361040:
Waiting for close lock at 1731750230726Disabling compacts and flushes for region at 1731750230726Disabling writes for close at 1731750230726Writing region close event to WAL at 1731750230726Closed at 1731750230726
2024-11-16T09:43:50,727 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b464e746ae538e2c12cc73c2da3bc8b3, NAME => 'testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.', STARTKEY => 'c', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b
2024-11-16T09:43:50,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741843_1019 (size=66)
2024-11-16T09:43:51,124 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,124 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1722): Closing b7087517245861969bfc8373d6d025e3, disabling compactions & flushes
2024-11-16T09:43:51,124 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,125 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,125 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3. after waiting 0 ms
2024-11-16T09:43:51,125 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,125 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,125 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1676): Region close journal for b7087517245861969bfc8373d6d025e3:
Waiting for close lock at 1731750231124Disabling compacts and flushes for region at 1731750231124Disabling writes for close at 1731750231125 (+1 ms)Writing region close event to WAL at 1731750231125Closed at 1731750231125
2024-11-16T09:43:51,126 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,126 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing 7db3eb8b266db3675f5937e0dbfbe0e9, disabling compactions & flushes
2024-11-16T09:43:51,126 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,126 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,126 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9. after waiting 0 ms
2024-11-16T09:43:51,126 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,126 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,126 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7db3eb8b266db3675f5937e0dbfbe0e9:
Waiting for close lock at 1731750231126Disabling compacts and flushes for region at 1731750231126Disabling writes for close at 1731750231126Writing region close event to WAL at 1731750231126Closed at 1731750231126
2024-11-16T09:43:51,136 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,136 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing b464e746ae538e2c12cc73c2da3bc8b3, disabling compactions & flushes
2024-11-16T09:43:51,136 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,136 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,136 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3. after waiting 0 ms
2024-11-16T09:43:51,136 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,136 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,137 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for b464e746ae538e2c12cc73c2da3bc8b3:
Waiting for close lock at 1731750231136Disabling compacts and flushes for region at 1731750231136Disabling writes for close at 1731750231136Writing region close event to WAL at 1731750231136Closed at 1731750231136
2024-11-16T09:43:51,138 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T09:43:51,139 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731750231139"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750231139"}]},"ts":"1731750231139"}
2024-11-16T09:43:51,139 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731750231139"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750231139"}]},"ts":"1731750231139"}
2024-11-16T09:43:51,139 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731750231139"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750231139"}]},"ts":"1731750231139"}
2024-11-16T09:43:51,139 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731750231139"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750231139"}]},"ts":"1731750231139"}
2024-11-16T09:43:51,144 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 4 regions to meta.
2024-11-16T09:43:51,146 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T09:43:51,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750231146"}]},"ts":"1731750231146"}
2024-11-16T09:43:51,148 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLING in hbase:meta
2024-11-16T09:43:51,149 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, ASSIGN}, {pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, ASSIGN}, {pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, ASSIGN}, {pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, ASSIGN}]
2024-11-16T09:43:51,152 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, ASSIGN
2024-11-16T09:43:51,153 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, ASSIGN; state=OFFLINE, location=037c58a19ddb,36121,1731750222645; forceNewPlan=false, retain=false
2024-11-16T09:43:51,154 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, ASSIGN
2024-11-16T09:43:51,155 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, ASSIGN; state=OFFLINE, location=037c58a19ddb,36121,1731750222645; forceNewPlan=false, retain=false
2024-11-16T09:43:51,156 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, ASSIGN
2024-11-16T09:43:51,156 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, ASSIGN; state=OFFLINE, location=037c58a19ddb,36121,1731750222645; forceNewPlan=false, retain=false
2024-11-16T09:43:51,158 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, ASSIGN
2024-11-16T09:43:51,158 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, ASSIGN; state=OFFLINE, location=037c58a19ddb,36121,1731750222645; forceNewPlan=false, retain=false
2024-11-16T09:43:51,303 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,304 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,305 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,305 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPENING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, ASSIGN because future has completed
2024-11-16T09:43:51,307 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:51,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, ASSIGN because future has completed
2024-11-16T09:43:51,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, ASSIGN because future has completed
2024-11-16T09:43:51,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, ASSIGN because future has completed
2024-11-16T09:43:51,312 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:51,315 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:51,319 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:51,397 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T09:43:51,465 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,466 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => b464e746ae538e2c12cc73c2da3bc8b3, NAME => 'testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.', STARTKEY => 'c', ENDKEY => ''}
2024-11-16T09:43:51,466 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,466 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,466 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,466 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,468 INFO  [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region b464e746ae538e2c12cc73c2da3bc8b3 
2024-11-16T09:43:51,470 INFO  [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b464e746ae538e2c12cc73c2da3bc8b3 columnFamilyName f1
2024-11-16T09:43:51,470 DEBUG [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,471 INFO  [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] regionserver.HStore(327): Store=b464e746ae538e2c12cc73c2da3bc8b3/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,471 INFO  [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region b464e746ae538e2c12cc73c2da3bc8b3 
2024-11-16T09:43:51,473 INFO  [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b464e746ae538e2c12cc73c2da3bc8b3 columnFamilyName f2
2024-11-16T09:43:51,473 DEBUG [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,473 INFO  [StoreOpener-b464e746ae538e2c12cc73c2da3bc8b3-1 {}] regionserver.HStore(327): Store=b464e746ae538e2c12cc73c2da3bc8b3/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,474 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,475 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,475 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,476 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,476 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,477 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-16T09:43:51,478 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,482 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:51,482 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1114): Opened b464e746ae538e2c12cc73c2da3bc8b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62345238, jitterRate=-0.07098355889320374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-16T09:43:51,482 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:51,484 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for b464e746ae538e2c12cc73c2da3bc8b3:
Running coprocessor pre-open hook at 1731750231466Writing region info on filesystem at 1731750231466Initializing all the Stores at 1731750231468 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231468Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231468Cleaning up temporary data from old regions at 1731750231476 (+8 ms)Running coprocessor post-open hooks at 1731750231482 (+6 ms)Region opened successfully at 1731750231483 (+1 ms)
2024-11-16T09:43:51,485 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3., pid=27, masterSystemTime=1731750231460
2024-11-16T09:43:51,488 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,488 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:51,488 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,489 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => b7087517245861969bfc8373d6d025e3, NAME => 'testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.', STARTKEY => 'b', ENDKEY => 'c'}
2024-11-16T09:43:51,489 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,489 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,489 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,489 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,489 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,491 INFO  [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region b7087517245861969bfc8373d6d025e3 
2024-11-16T09:43:51,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:51,493 INFO  [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7087517245861969bfc8373d6d025e3 columnFamilyName f1
2024-11-16T09:43:51,494 DEBUG [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,494 INFO  [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] regionserver.HStore(327): Store=b7087517245861969bfc8373d6d025e3/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,494 INFO  [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region b7087517245861969bfc8373d6d025e3 
2024-11-16T09:43:51,497 INFO  [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7087517245861969bfc8373d6d025e3 columnFamilyName f2
2024-11-16T09:43:51,497 DEBUG [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26
2024-11-16T09:43:51,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 in 187 msec
2024-11-16T09:43:51,497 INFO  [StoreOpener-b7087517245861969bfc8373d6d025e3-1 {}] regionserver.HStore(327): Store=b7087517245861969bfc8373d6d025e3/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,498 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,499 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,499 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, ASSIGN in 348 msec
2024-11-16T09:43:51,500 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,501 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,501 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,502 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-16T09:43:51,504 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,507 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:51,508 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened b7087517245861969bfc8373d6d025e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62143888, jitterRate=-0.07398390769958496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-16T09:43:51,508 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:51,509 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for b7087517245861969bfc8373d6d025e3:
Running coprocessor pre-open hook at 1731750231489Writing region info on filesystem at 1731750231489Initializing all the Stores at 1731750231491 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231491Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231491Cleaning up temporary data from old regions at 1731750231501 (+10 ms)Running coprocessor post-open hooks at 1731750231509 (+8 ms)Region opened successfully at 1731750231509
2024-11-16T09:43:51,510 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3., pid=30, masterSystemTime=1731750231460
2024-11-16T09:43:51,513 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,513 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:51,513 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,513 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 7db3eb8b266db3675f5937e0dbfbe0e9, NAME => 'testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.', STARTKEY => '', ENDKEY => 'a'}
2024-11-16T09:43:51,513 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,513 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,514 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,514 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,514 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,516 INFO  [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 7db3eb8b266db3675f5937e0dbfbe0e9 
2024-11-16T09:43:51,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:51,518 INFO  [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7db3eb8b266db3675f5937e0dbfbe0e9 columnFamilyName f1
2024-11-16T09:43:51,518 DEBUG [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,519 INFO  [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] regionserver.HStore(327): Store=7db3eb8b266db3675f5937e0dbfbe0e9/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,519 INFO  [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 7db3eb8b266db3675f5937e0dbfbe0e9 
2024-11-16T09:43:51,521 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=25
2024-11-16T09:43:51,521 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 in 199 msec
2024-11-16T09:43:51,521 INFO  [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7db3eb8b266db3675f5937e0dbfbe0e9 columnFamilyName f2
2024-11-16T09:43:51,522 DEBUG [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,522 INFO  [StoreOpener-7db3eb8b266db3675f5937e0dbfbe0e9-1 {}] regionserver.HStore(327): Store=7db3eb8b266db3675f5937e0dbfbe0e9/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,522 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,524 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,524 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, ASSIGN in 372 msec
2024-11-16T09:43:51,525 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,526 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,526 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,527 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-16T09:43:51,528 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,531 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:51,532 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 7db3eb8b266db3675f5937e0dbfbe0e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65767066, jitterRate=-0.019994348287582397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-16T09:43:51,532 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:51,532 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 7db3eb8b266db3675f5937e0dbfbe0e9:
Running coprocessor pre-open hook at 1731750231514Writing region info on filesystem at 1731750231514Initializing all the Stores at 1731750231515 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231516 (+1 ms)Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231516Cleaning up temporary data from old regions at 1731750231526 (+10 ms)Running coprocessor post-open hooks at 1731750231532 (+6 ms)Region opened successfully at 1731750231532
2024-11-16T09:43:51,534 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9., pid=28, masterSystemTime=1731750231460
2024-11-16T09:43:51,536 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,536 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:51,537 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:51,537 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => 416cf44c077ffd5b666389825d361040, NAME => 'testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.', STARTKEY => 'a', ENDKEY => 'b'}
2024-11-16T09:43:51,537 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,538 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,538 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T09:43:51,538 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,538 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,540 INFO  [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 416cf44c077ffd5b666389825d361040 
2024-11-16T09:43:51,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:51,543 INFO  [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 416cf44c077ffd5b666389825d361040 columnFamilyName f1
2024-11-16T09:43:51,543 DEBUG [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,543 INFO  [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] regionserver.HStore(327): Store=416cf44c077ffd5b666389825d361040/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,544 INFO  [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 416cf44c077ffd5b666389825d361040 
2024-11-16T09:43:51,545 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=23
2024-11-16T09:43:51,545 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 in 230 msec
2024-11-16T09:43:51,545 INFO  [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 416cf44c077ffd5b666389825d361040 columnFamilyName f2
2024-11-16T09:43:51,546 DEBUG [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T09:43:51,546 INFO  [StoreOpener-416cf44c077ffd5b666389825d361040-1 {}] regionserver.HStore(327): Store=416cf44c077ffd5b666389825d361040/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T09:43:51,546 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,547 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,547 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, ASSIGN in 396 msec
2024-11-16T09:43:51,548 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,549 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,549 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,549 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-16T09:43:51,551 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,554 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T09:43:51,554 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened 416cf44c077ffd5b666389825d361040; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62183022, jitterRate=-0.07340076565742493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-16T09:43:51,555 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:51,555 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for 416cf44c077ffd5b666389825d361040:
Running coprocessor pre-open hook at 1731750231538Writing region info on filesystem at 1731750231538Initializing all the Stores at 1731750231540 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231540Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750231540Cleaning up temporary data from old regions at 1731750231549 (+9 ms)Running coprocessor post-open hooks at 1731750231555 (+6 ms)Region opened successfully at 1731750231555
2024-11-16T09:43:51,556 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040., pid=29, masterSystemTime=1731750231460
2024-11-16T09:43:51,558 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:51,558 INFO  [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:51,559 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:51,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:51,567 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=24
2024-11-16T09:43:51,567 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 in 249 msec
2024-11-16T09:43:51,571 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22
2024-11-16T09:43:51,571 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, ASSIGN in 419 msec
2024-11-16T09:43:51,573 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T09:43:51,573 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750231573"}]},"ts":"1731750231573"}
2024-11-16T09:43:51,576 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLED in hbase:meta
2024-11-16T09:43:51,577 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T09:43:51,580 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution in 1.2840 sec
2024-11-16T09:43:51,796 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true
2024-11-16T09:43:51,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:51,799 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:51,799 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:51,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:51,800 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:52,044 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-16T09:43:52,048 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,049 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:52,049 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,049 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:52,050 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:52,050 ERROR [Time-limited test {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[HFileArchiver-1,5,PEWorkerGroup]
    Thread[HFileArchiver-2,5,PEWorkerGroup]
2024-11-16T09:43:52,051 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:52,051 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:52,051 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:52,051 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:52,052 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:52,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[]
2024-11-16T09:43:52,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:52,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:52,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:52,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:52,063 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec
2024-11-16T09:43:52,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,066 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:52,066 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:52,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,071 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750262070
2024-11-16T09:43:52,071 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:52,071 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,077 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:52,077 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750292077
2024-11-16T09:43:52,077 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750352077
2024-11-16T09:43:52,077 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec
2024-11-16T09:43:52,078 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:52,079 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:52,079 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,079 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,079 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:52,328 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:52,329 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:52,330 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:52,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0]
2024-11-16T09:43:52,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:52,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:52,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:52,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:52,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:52,335 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:52,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,336 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:52,336 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:52,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,339 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750262339
2024-11-16T09:43:52,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:52,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,346 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:52,346 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750292346
2024-11-16T09:43:52,346 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750352346
2024-11-16T09:43:52,346 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec
2024-11-16T09:43:52,346 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:52,347 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750232347"}]},"ts":"1731750232347"}
2024-11-16T09:43:52,350 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta
2024-11-16T09:43:52,350 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING
2024-11-16T09:43:52,350 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:52,350 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,350 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,350 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:52,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T09:43:52,373 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-16T09:43:52,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T09:43:52,375 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-16T09:43:52,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-16T09:43:52,375 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-16T09:43:52,376 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution
2024-11-16T09:43:52,376 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution Metrics about Tables on a single HBase RegionServer
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:52,596 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:52,597 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:52,597 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:52,598 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:52,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1]
2024-11-16T09:43:52,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:52,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:52,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:52,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:52,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:52,603 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:52,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,604 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:52,604 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:52,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,607 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750262607
2024-11-16T09:43:52,607 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,607 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:52,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,612 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:52,612 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750292612
2024-11-16T09:43:52,612 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750352612
2024-11-16T09:43:52,612 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-16T09:43:52,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750232612"}]},"ts":"1731750232612"}
2024-11-16T09:43:52,615 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta
2024-11-16T09:43:52,615 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING
2024-11-16T09:43:52,615 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:52,616 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}]
2024-11-16T09:43:52,616 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:52,616 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,616 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,617 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:52,862 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,862 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:52,862 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,862 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:52,862 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:52,863 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:52,863 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:52,863 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:52,863 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:52,864 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:52,866 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2]
2024-11-16T09:43:52,866 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:52,867 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:52,867 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:52,867 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:52,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:52,869 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,869 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,869 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:52,869 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:52,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,870 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:52,870 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:52,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:52,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:52,873 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750262873
2024-11-16T09:43:52,873 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:52,873 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:52,879 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:52,879 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750292879
2024-11-16T09:43:52,879 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750352879
2024-11-16T09:43:52,879 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-16T09:43:52,879 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}]
2024-11-16T09:43:52,879 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:52,882 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN}]
2024-11-16T09:43:52,882 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:52,882 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:52,882 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:52,882 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:53,129 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:53,130 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:53,131 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:53,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:53,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[]
2024-11-16T09:43:53,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:53,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:53,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:53,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:53,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,135 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,135 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,135 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:53,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,136 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:53,137 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:53,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,139 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750263139
2024-11-16T09:43:53,139 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:53,139 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,144 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,145 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:53,145 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750293145
2024-11-16T09:43:53,145 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750353145
2024-11-16T09:43:53,145 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-16T09:43:53,145 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN}]
2024-11-16T09:43:53,145 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:53,149 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,150 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,151 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,152 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,153 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:53,154 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,155 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:53,155 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:53,155 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:53,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:53,155 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,155 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,155 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:53,155 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fa5c8bb9c78@24d023e9 rejected from java.util.concurrent.ThreadPoolExecutor@30ff3c08[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:53,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN because future has completed
2024-11-16T09:43:53,157 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fa5c8bb9c78@32190fad rejected from java.util.concurrent.ThreadPoolExecutor@30ff3c08[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:53,250 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,250 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,353 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,353 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,395 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,395 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:53,395 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,395 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:53,395 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:53,396 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:53,396 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:53,396 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:53,396 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:53,397 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:53,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:53,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:53,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:53,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,403 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,403 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,403 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,403 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,403 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-16T09:43:53,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,404 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:53,404 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:53,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,407 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9 to restore RIT
2024-11-16T09:43:53,407 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040 to restore RIT
2024-11-16T09:43:53,407 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:53,407 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:53,408 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750263408
2024-11-16T09:43:53,408 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:53,408 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,408 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,408 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,408 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,412 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:53,412 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750293412
2024-11-16T09:43:53,412 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750353412
2024-11-16T09:43:53,412 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:53,412 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,414 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,414 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,415 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,415 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN because future has completed
2024-11-16T09:43:53,417 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:53,417 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:53,417 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:53,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN because future has completed
2024-11-16T09:43:53,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:53,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:53,421 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:53,421 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:53,421 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:53,421 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,421 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,421 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:53,456 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,456 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,559 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,559 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,570 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(122): Close 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:53,570 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-16T09:43:53,570 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1722): Closing 7db3eb8b266db3675f5937e0dbfbe0e9, disabling compactions & flushes
2024-11-16T09:43:53,570 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:53,570 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:53,570 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9. after waiting 0 ms
2024-11-16T09:43:53,570 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:53,576 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-16T09:43:53,577 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.
2024-11-16T09:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1676): Region close journal for 7db3eb8b266db3675f5937e0dbfbe0e9:
Waiting for close lock at 1731750233570Running coprocessor pre-close hooks at 1731750233570Disabling compacts and flushes for region at 1731750233570Disabling writes for close at 1731750233570Writing region close event to WAL at 1731750233571 (+1 ms)Running coprocessor post-close hooks at 1731750233577 (+6 ms)Closed at 1731750233577
2024-11-16T09:43:53,580 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(157): Closed 7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:53,661 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,662 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,662 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,662 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:53,662 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,663 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:53,663 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:53,663 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:53,663 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:53,663 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:53,663 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:53,664 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:53,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:53,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:53,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:53,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,670 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,670 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,670 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,670 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,670 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:53,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,671 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:53,671 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:53,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,674 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9 to restore RIT
2024-11-16T09:43:53,674 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040 to restore RIT
2024-11-16T09:43:53,674 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:53,674 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:53,677 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750263677
2024-11-16T09:43:53,677 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:53,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,682 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:53,682 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750293682
2024-11-16T09:43:53,682 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750353682
2024-11-16T09:43:53,682 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:53,683 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED
2024-11-16T09:43:53,685 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,685 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:53,686 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN because future has completed
2024-11-16T09:43:53,687 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:53,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:53,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:53,690 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=33
2024-11-16T09:43:53,690 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 in 270 msec
2024-11-16T09:43:53,690 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:53,690 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,690 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,690 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:53,764 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,764 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,764 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,867 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,867 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,867 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:53,932 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:53,933 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:53,934 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:53,936 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:53,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:53,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,939 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,940 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,940 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:53,940 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:53,940 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:53,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,941 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:53,941 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:53,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:53,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,943 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:53,943 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9 to restore RIT
2024-11-16T09:43:53,944 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040 to restore RIT
2024-11-16T09:43:53,944 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:53,944 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:53,944 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750263944
2024-11-16T09:43:53,944 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:53,944 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,944 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,944 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:53,944 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:53,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:53,949 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:53,949 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750293949
2024-11-16T09:43:53,949 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750353949
2024-11-16T09:43:53,949 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-16T09:43:53,949 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:53,951 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:53,952 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN in 806 msec
2024-11-16T09:43:53,952 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,953 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,953 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:53,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN because future has completed
2024-11-16T09:43:53,955 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:53,956 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:53,956 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:53,956 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:53,956 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:53,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:53,956 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:53,956 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fa5c8bb9c78@15888c0c rejected from java.util.concurrent.ThreadPoolExecutor@66793080[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:53,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:53,957 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fa5c8bb9c78@70387e09 rejected from java.util.concurrent.ThreadPoolExecutor@66793080[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:53,969 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:53,970 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,072 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,072 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,175 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,175 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,199 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:54,200 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:54,201 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:54,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:54,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:54,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:54,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,207 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,207 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,207 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,207 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:54,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,208 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:54,208 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:54,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:54,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:54,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:54,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:54,210 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040 to restore RIT
2024-11-16T09:43:54,211 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:54,211 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:54,211 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750264211
2024-11-16T09:43:54,211 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:54,211 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,211 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,212 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,216 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:54,216 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750294216
2024-11-16T09:43:54,216 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750354216
2024-11-16T09:43:54,216 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:54,216 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,217 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,218 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN because future has completed
2024-11-16T09:43:54,219 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:54,220 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:54,220 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:54,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:54,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:54,224 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:54,224 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:54,224 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:54,224 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,224 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,224 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:54,278 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,374 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:54,374 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-16T09:43:54,374 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 416cf44c077ffd5b666389825d361040, disabling compactions & flushes
2024-11-16T09:43:54,374 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:54,374 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:54,374 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040. after waiting 0 ms
2024-11-16T09:43:54,374 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:54,380 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-16T09:43:54,380 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,381 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.
2024-11-16T09:43:54,381 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 416cf44c077ffd5b666389825d361040:
Waiting for close lock at 1731750234374Running coprocessor pre-close hooks at 1731750234374Disabling compacts and flushes for region at 1731750234374Disabling writes for close at 1731750234374Writing region close event to WAL at 1731750234374Running coprocessor post-close hooks at 1731750234381 (+7 ms)Closed at 1731750234381
2024-11-16T09:43:54,384 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 416cf44c077ffd5b666389825d361040
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 9 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:54,466 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:54,467 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:54,469 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:54,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:54,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:54,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:54,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:54,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9]
2024-11-16T09:43:54,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10]
2024-11-16T09:43:54,471 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,471 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:54,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:54,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:54,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:54,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,472 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,474 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,474 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,474 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,474 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:54,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,475 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:54,475 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:54,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:54,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:54,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:54,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:54,477 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040 to restore RIT
2024-11-16T09:43:54,477 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:54,477 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:54,478 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750264478
2024-11-16T09:43:54,478 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:54,478 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,478 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,478 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,482 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:54,482 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750294482
2024-11-16T09:43:54,482 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750354482
2024-11-16T09:43:54,482 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:54,484 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=416cf44c077ffd5b666389825d361040, regionState=CLOSED
2024-11-16T09:43:54,486 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:54,486 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,488 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:54,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:54,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:54,491 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=34
2024-11-16T09:43:54,491 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 in 268 msec
2024-11-16T09:43:54,491 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:54,491 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,491 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,492 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:54,586 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,586 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,689 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,689 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,732 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 10 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,732 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:54,732 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,733 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:54,733 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:54,733 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:54,733 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:54,733 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:54,733 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:54,735 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,736 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:54,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:54,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:54,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:54,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:54,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:54,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,739 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,740 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:54,740 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:54,740 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:54,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,741 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:54,741 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:54,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:54,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:54,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:54,743 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:54,743 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040 to restore RIT
2024-11-16T09:43:54,743 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:54,744 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:54,744 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750264744
2024-11-16T09:43:54,744 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:54,744 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,744 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,744 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:54,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:54,749 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:54,749 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750294749
2024-11-16T09:43:54,749 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750354749
2024-11-16T09:43:54,749 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:54,749 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:54,750 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:54,752 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN in 1.6050 sec
2024-11-16T09:43:54,752 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,753 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:54,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:54,756 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:54,756 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:54,756 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:54,756 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:54,756 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,756 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:54,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:54,757 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007fa5c8bb9c78@3ebbad96 rejected from java.util.concurrent.ThreadPoolExecutor@56c76f86[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T09:43:54,792 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,894 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,997 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 11 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:54,999 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:55,000 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:55,000 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:55,001 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:55,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:55,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:55,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:55,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:55,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,005 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:55,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:55,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:55,007 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:55,007 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,007 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,007 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-16T09:43:55,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,008 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:55,008 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:55,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,010 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,011 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:55,011 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:55,011 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750265011
2024-11-16T09:43:55,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:55,011 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,011 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,015 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:55,015 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750295015
2024-11-16T09:43:55,015 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750355015
2024-11-16T09:43:55,015 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:55,015 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,016 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN because future has completed
2024-11-16T09:43:55,018 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:55,019 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:55,019 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:55,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:55,023 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:55,023 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:55,023 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:55,023 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,023 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,023 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:55,172 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(122): Close b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:55,172 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-16T09:43:55,172 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1722): Closing b7087517245861969bfc8373d6d025e3, disabling compactions & flushes
2024-11-16T09:43:55,172 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:55,172 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:55,172 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3. after waiting 0 ms
2024-11-16T09:43:55,173 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:55,179 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-16T09:43:55,179 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.
2024-11-16T09:43:55,179 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1676): Region close journal for b7087517245861969bfc8373d6d025e3:
Waiting for close lock at 1731750235172Running coprocessor pre-close hooks at 1731750235172Disabling compacts and flushes for region at 1731750235172Disabling writes for close at 1731750235172Writing region close event to WAL at 1731750235173 (+1 ms)Running coprocessor post-close hooks at 1731750235179 (+6 ms)Closed at 1731750235179
2024-11-16T09:43:55,182 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(157): Closed b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:55,265 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 12 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,265 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:55,265 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,265 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:55,266 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:55,266 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:55,266 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:55,266 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:55,266 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:55,267 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:55,269 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:55,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:55,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:55,270 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:55,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,271 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,272 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:55,273 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:55,273 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,273 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:55,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,274 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:55,274 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:55,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,277 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:55,277 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:55,277 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750265277
2024-11-16T09:43:55,277 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:55,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,281 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:55,282 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750295281
2024-11-16T09:43:55,282 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750355282
2024-11-16T09:43:55,282 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-16T09:43:55,283 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b7087517245861969bfc8373d6d025e3, regionState=CLOSED
2024-11-16T09:43:55,285 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:55,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:55,287 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:55,289 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=35
2024-11-16T09:43:55,289 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 in 268 msec
2024-11-16T09:43:55,289 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:55,289 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,289 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,290 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:55,303 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:55,405 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:55,508 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 13 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:55,532 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:55,533 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:55,534 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:55,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:55,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:55,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:55,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:55,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13]
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:55,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:55,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:55,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:55,539 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,539 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:55,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,540 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:55,540 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:55,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,543 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3 to restore RIT
2024-11-16T09:43:55,543 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:55,543 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750265543
2024-11-16T09:43:55,543 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:55,544 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,544 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,547 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:55,547 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750295547
2024-11-16T09:43:55,547 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750355547
2024-11-16T09:43:55,547 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:55,547 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,549 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:55,551 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN in 2.4040 sec
2024-11-16T09:43:55,551 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:55,553 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:55,554 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:55,554 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:55,554 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,554 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,554 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:55,797 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 14 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,797 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:55,797 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,797 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:55,798 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:55,798 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:55,798 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:55,798 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:55,798 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:55,799 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-16T09:43:55,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:55,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:55,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:55,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:55,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,803 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:55,803 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,804 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:55,805 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,805 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,805 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,805 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:55,805 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:55,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,805 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:55,806 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:55,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:55,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:55,808 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:55,808 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750265808
2024-11-16T09:43:55,808 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:55,809 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:55,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:55,812 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:55,812 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750295812
2024-11-16T09:43:55,812 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750355812
2024-11-16T09:43:55,812 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:55,813 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSING, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,815 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37567 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=037c58a19ddb,36121,1731750222645, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-16T09:43:55,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN because future has completed
2024-11-16T09:43:55,816 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-16T09:43:55,816 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645}]
2024-11-16T09:43:55,816 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:55,969 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:55,969 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-16T09:43:55,969 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing b464e746ae538e2c12cc73c2da3bc8b3, disabling compactions & flushes
2024-11-16T09:43:55,969 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:55,969 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:55,969 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3. after waiting 0 ms
2024-11-16T09:43:55,969 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:55,975 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-16T09:43:55,976 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.
2024-11-16T09:43:55,976 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for b464e746ae538e2c12cc73c2da3bc8b3:
Waiting for close lock at 1731750235969Running coprocessor pre-close hooks at 1731750235969Disabling compacts and flushes for region at 1731750235969Disabling writes for close at 1731750235969Writing region close event to WAL at 1731750235970 (+1 ms)Running coprocessor post-close hooks at 1731750235975 (+5 ms)Closed at 1731750235976 (+1 ms)
2024-11-16T09:43:55,978 INFO  [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:55,979 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED
2024-11-16T09:43:55,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:55,982 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:55,982 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:55,982 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:55,982 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 15 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:56,063 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:56,064 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17]
2024-11-16T09:43:56,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18]
2024-11-16T09:43:56,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:56,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:56,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:56,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:56,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:56,070 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:56,070 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,070 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:56,071 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:56,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,073 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:56,074 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750266074
2024-11-16T09:43:56,074 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:56,074 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,078 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:56,078 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750296078
2024-11-16T09:43:56,078 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750356078
2024-11-16T09:43:56,078 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:56,078 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, regionLocation=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 because future has completed
2024-11-16T09:43:56,082 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:56,084 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=36
2024-11-16T09:43:56,084 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 in 266 msec
2024-11-16T09:43:56,084 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:56,084 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,084 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,085 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 16 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:56,328 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:56,329 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:56,330 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:56,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:56,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:56,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:56,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,334 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:56,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:56,335 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:56,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,338 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3 to restore RIT
2024-11-16T09:43:56,338 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750266338
2024-11-16T09:43:56,338 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:56,338 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,342 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:56,342 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750296342
2024-11-16T09:43:56,342 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750356342
2024-11-16T09:43:56,342 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:56,342 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,343 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:56,346 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=32
2024-11-16T09:43:56,346 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN in 3.1980 sec
2024-11-16T09:43:56,347 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:56,347 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,347 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,347 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 17 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:56,592 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:56,593 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:56,594 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-16T09:43:56,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:56,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:56,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:56,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:56,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,598 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:56,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,599 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:56,599 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:56,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,602 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750266602
2024-11-16T09:43:56,602 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:56,602 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,606 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:56,606 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750296606
2024-11-16T09:43:56,606 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750356606
2024-11-16T09:43:56,606 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:56,606 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:56,609 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31
2024-11-16T09:43:56,609 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution in 3.7270 sec
2024-11-16T09:43:56,610 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:56,610 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,610 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,610 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 18 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:56,856 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:56,857 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:56,858 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-16T09:43:56,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:56,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:56,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:56,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:56,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:56,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:56,862 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:56,863 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,863 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:56,863 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:56,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:56,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:56,866 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750266866
2024-11-16T09:43:56,866 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:56,866 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:56,870 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:56,870 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750296870
2024-11-16T09:43:56,870 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750356870
2024-11-16T09:43:56,870 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:56,870 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:56,871 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750236871"}]},"ts":"1731750236871"}
2024-11-16T09:43:56,874 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta
2024-11-16T09:43:56,874 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED
2024-11-16T09:43:56,874 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:56,874 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:56,874 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:56,874 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:56,968 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-16T09:43:57,120 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 19 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,120 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:57,120 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:57,120 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:57,120 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:57,121 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:57,121 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:57,121 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:57,121 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:57,122 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-16T09:43:57,123 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:57,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:57,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:57,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:57,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:57,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:57,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:57,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,126 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,127 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T09:43:57,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,127 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:57,127 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:57,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:57,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:57,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:57,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:57,130 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750267130
2024-11-16T09:43:57,130 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:57,130 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,134 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:57,134 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750297134
2024-11-16T09:43:57,134 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750357134
2024-11-16T09:43:57,134 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:57,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750237134"}]},"ts":"1731750237134"}
2024-11-16T09:43:57,137 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta
2024-11-16T09:43:57,137 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED
2024-11-16T09:43:57,137 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:57,139 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-16T09:43:57,139 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,139 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:57,139 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:57,384 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 20 exec state=pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,384 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-16T09:43:57,384 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:57,384 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:57,384 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:57,385 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-16T09:43:57,385 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-16T09:43:57,385 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-16T09:43:57,385 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T09:43:57,386 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22, 23]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-16T09:43:57,387 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T09:43:57,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-16T09:43:57,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-16T09:43:57,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-16T09:43:57,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:57,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:57,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-16T09:43:57,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN
2024-11-16T09:43:57,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN
2024-11-16T09:43:57,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=7db3eb8b266db3675f5937e0dbfbe0e9, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=416cf44c077ffd5b666389825d361040, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b7087517245861969bfc8373d6d025e3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b464e746ae538e2c12cc73c2da3bc8b3, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 7db3eb8b266db3675f5937e0dbfbe0e9, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 416cf44c077ffd5b666389825d361040, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b7087517245861969bfc8373d6d025e3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure b464e746ae538e2c12cc73c2da3bc8b3, server=037c58a19ddb,36121,1731750222645 didn't hold the lock before restarting, skip acquiring lock.
2024-11-16T09:43:57,390 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec
2024-11-16T09:43:57,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,391 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=037c58a19ddb,36121,1731750222645, table=hbase:meta, region=1588230740
2024-11-16T09:43:57,391 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,36121,1731750222645, state=OPEN
2024-11-16T09:43:57,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:57,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T09:43:57,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:57,393 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T09:43:57,393 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750267393
2024-11-16T09:43:57,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-16T09:43:57,394 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,396 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7db3eb8b266db3675f5937e0dbfbe0e9, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=416cf44c077ffd5b666389825d361040, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7087517245861969bfc8373d6d025e3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b464e746ae538e2c12cc73c2da3bc8b3, regionState=CLOSED, lastHost=037c58a19ddb,36121,1731750222645, regionLocation=037c58a19ddb,36121,1731750222645, openSeqNum=2
2024-11-16T09:43:57,397 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-16T09:43:57,397 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750297397
2024-11-16T09:43:57,397 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750357397
2024-11-16T09:43:57,397 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-16T09:43:57,397 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-16T09:43:57,400 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution in 5.6010 sec
2024-11-16T09:43:57,647 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-16T09:43:57,649 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,650 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testRecoveryAndDoubleExecution
2024-11-16T09:43:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-16T09:43:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] ipc.CallRunner(138): callId: 12 service: MasterService methodName: DisableTable size: 92 connection: 172.17.0.2:54140 deadline: 1731750297650, exception=org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
2024-11-16T09:43:57,651 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testRecoveryAndDoubleExecution failed with org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-16T09:43:57,652 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testRecoveryAndDoubleExecution already disabled, so just deleting it.
2024-11-16T09:43:57,652 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testRecoveryAndDoubleExecution
2024-11-16T09:43:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-16T09:43:57,656 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:57,664 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040
2024-11-16T09:43:57,664 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:57,664 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:57,668 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/f1, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/f2, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/recovered.edits]
2024-11-16T09:43:57,668 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/f1, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/f2, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/recovered.edits]
2024-11-16T09:43:57,668 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/f1, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/f2, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/recovered.edits]
2024-11-16T09:43:57,668 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/f1, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/f2, FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/recovered.edits]
2024-11-16T09:43:57,680 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/recovered.edits/4.seqid to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/archive/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040/recovered.edits/4.seqid
2024-11-16T09:43:57,680 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/recovered.edits/4.seqid to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/archive/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9/recovered.edits/4.seqid
2024-11-16T09:43:57,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/recovered.edits/4.seqid to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/archive/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3/recovered.edits/4.seqid
2024-11-16T09:43:57,681 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/recovered.edits/4.seqid to hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/archive/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3/recovered.edits/4.seqid
2024-11-16T09:43:57,681 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/7db3eb8b266db3675f5937e0dbfbe0e9
2024-11-16T09:43:57,681 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/416cf44c077ffd5b666389825d361040
2024-11-16T09:43:57,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b464e746ae538e2c12cc73c2da3bc8b3
2024-11-16T09:43:57,682 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/default/testRecoveryAndDoubleExecution/b7087517245861969bfc8373d6d025e3
2024-11-16T09:43:57,682 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testRecoveryAndDoubleExecution regions
2024-11-16T09:43:57,685 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,689 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 4 rows of testRecoveryAndDoubleExecution from hbase:meta
2024-11-16T09:43:57,692 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testRecoveryAndDoubleExecution' descriptor.
2024-11-16T09:43:57,694 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,694 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testRecoveryAndDoubleExecution' from region states.
2024-11-16T09:43:57,695 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731750237694"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:57,695 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731750237694"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:57,695 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731750237694"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:57,695 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731750237694"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:57,699 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 4 regions from META
2024-11-16T09:43:57,699 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7db3eb8b266db3675f5937e0dbfbe0e9, NAME => 'testRecoveryAndDoubleExecution,,1731750230292.7db3eb8b266db3675f5937e0dbfbe0e9.', STARTKEY => '', ENDKEY => 'a'}, {ENCODED => 416cf44c077ffd5b666389825d361040, NAME => 'testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040.', STARTKEY => 'a', ENDKEY => 'b'}, {ENCODED => b7087517245861969bfc8373d6d025e3, NAME => 'testRecoveryAndDoubleExecution,b,1731750230292.b7087517245861969bfc8373d6d025e3.', STARTKEY => 'b', ENDKEY => 'c'}, {ENCODED => b464e746ae538e2c12cc73c2da3bc8b3, NAME => 'testRecoveryAndDoubleExecution,c,1731750230292.b464e746ae538e2c12cc73c2da3bc8b3.', STARTKEY => 'c', ENDKEY => ''}]
2024-11-16T09:43:57,699 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testRecoveryAndDoubleExecution' as deleted.
2024-11-16T09:43:57,699 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731750237699"}]},"ts":"9223372036854775807"}
2024-11-16T09:43:57,701 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testRecoveryAndDoubleExecution state from META
2024-11-16T09:43:57,702 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-16T09:43:57,703 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution in 50 msec
2024-11-16T09:43:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37567 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-16T09:43:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testRecoveryAndDoubleExecution
2024-11-16T09:43:57,769 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testRecoveryAndDoubleExecution completed
2024-11-16T09:43:57,780 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=219 (was 224), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=271 (was 294), ProcessCount=11 (was 11), AvailableMemoryMB=6059 (was 6087)
2024-11-16T09:43:57,780 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T09:43:57,781 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T09:43:57,781 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T09:43:57,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T09:43:57,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T09:43:57,789 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T09:43:57,789 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T09:43:57,789 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=61195404, stopped=false
2024-11-16T09:43:57,790 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=037c58a19ddb,37567,1731750221953
2024-11-16T09:43:57,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T09:43:57,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T09:43:57,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:57,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:57,791 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:57,791 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T09:43:57,792 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T09:43:57,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T09:43:57,792 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T09:43:57,792 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T09:43:57,792 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,36121,1731750222645' *****
2024-11-16T09:43:57,792 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T09:43:57,792 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T09:43:57,793 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,36121,1731750222645
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;037c58a19ddb:36121.
2024-11-16T09:43:57,793 DEBUG [RS:0;037c58a19ddb:36121 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T09:43:57,793 DEBUG [RS:0;037c58a19ddb:36121 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T09:43:57,793 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T09:43:57,794 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-16T09:43:57,794 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-16T09:43:57,794 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:57,794 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T09:43:57,794 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T09:43:57,795 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T09:43:57,795 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T09:43:57,795 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T09:43:57,798 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=25.47 KB heapSize=41.64 KB
2024-11-16T09:43:57,856 INFO  [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T09:43:57,863 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/info/9dcdb737bc4f478eb5844ca60bf044f7 is 108, key is testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040./info:/1731750237694/DeleteFamily/seqid=0
2024-11-16T09:43:57,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741844_1020 (size=6669)
2024-11-16T09:43:57,995 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:58,195 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:58,275 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.63 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/info/9dcdb737bc4f478eb5844ca60bf044f7
2024-11-16T09:43:58,346 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/ns/48ab46a42a5042158528e5bbdd6b5cde is 106, key is testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040./ns:/1731750237685/DeleteFamily/seqid=0
2024-11-16T09:43:58,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741845_1021 (size=6009)
2024-11-16T09:43:58,395 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:58,433 INFO  [regionserver/037c58a19ddb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-16T09:43:58,434 INFO  [regionserver/037c58a19ddb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-16T09:43:58,595 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:58,753 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/ns/48ab46a42a5042158528e5bbdd6b5cde
2024-11-16T09:43:58,779 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/rep_barrier/7a065e3c2750429285be10d55a5cd679 is 115, key is testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040./rep_barrier:/1731750237685/DeleteFamily/seqid=0
2024-11-16T09:43:58,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741846_1022 (size=6121)
2024-11-16T09:43:58,796 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:58,996 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:59,186 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=648 B at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/rep_barrier/7a065e3c2750429285be10d55a5cd679
2024-11-16T09:43:59,196 DEBUG [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T09:43:59,214 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/table/9262e62321e54834a192d4f07f6018d4 is 109, key is testRecoveryAndDoubleExecution,a,1731750230292.416cf44c077ffd5b666389825d361040./table:/1731750237685/DeleteFamily/seqid=0
2024-11-16T09:43:59,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741847_1023 (size=6158)
2024-11-16T09:43:59,228 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.55 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/table/9262e62321e54834a192d4f07f6018d4
2024-11-16T09:43:59,237 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/info/9dcdb737bc4f478eb5844ca60bf044f7 as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/info/9dcdb737bc4f478eb5844ca60bf044f7
2024-11-16T09:43:59,247 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/info/9dcdb737bc4f478eb5844ca60bf044f7, entries=12, sequenceid=78, filesize=6.5 K
2024-11-16T09:43:59,250 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/ns/48ab46a42a5042158528e5bbdd6b5cde as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/ns/48ab46a42a5042158528e5bbdd6b5cde
2024-11-16T09:43:59,259 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/ns/48ab46a42a5042158528e5bbdd6b5cde, entries=8, sequenceid=78, filesize=5.9 K
2024-11-16T09:43:59,260 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/rep_barrier/7a065e3c2750429285be10d55a5cd679 as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/rep_barrier/7a065e3c2750429285be10d55a5cd679
2024-11-16T09:43:59,269 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/rep_barrier/7a065e3c2750429285be10d55a5cd679, entries=6, sequenceid=78, filesize=6.0 K
2024-11-16T09:43:59,270 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/.tmp/table/9262e62321e54834a192d4f07f6018d4 as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/table/9262e62321e54834a192d4f07f6018d4
2024-11-16T09:43:59,280 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/table/9262e62321e54834a192d4f07f6018d4, entries=9, sequenceid=78, filesize=6.0 K
2024-11-16T09:43:59,286 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~25.47 KB/26078, heapSize ~41.58 KB/42576, currentSize=0 B/0 for 1588230740 in 1486ms, sequenceid=78, compaction requested=false
2024-11-16T09:43:59,294 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/data/hbase/meta/1588230740/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1
2024-11-16T09:43:59,295 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T09:43:59,295 INFO  [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T09:43:59,296 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731750237794Running coprocessor pre-close hooks at 1731750237794Disabling compacts and flushes for region at 1731750237794Disabling writes for close at 1731750237795 (+1 ms)Obtaining lock to block concurrent updates at 1731750237798 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1731750237798Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=26078, getHeapSize=42576, getOffHeapSize=0, getCellsCount=205 at 1731750237805 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731750237807 (+2 ms)Flushing 1588230740/info: creating writer at 1731750237810 (+3 ms)Flushing 1588230740/info: appending metadata at 1731750237852 (+42 ms)Flushing 1588230740/info: closing flushed file at 1731750237855 (+3 ms)Flushing 1588230740/ns: creating writer at 1731750238321 (+466 ms)Flushing 1588230740/ns: appending metadata at 1731750238345 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731750238345Flushing 1588230740/rep_barrier: creating writer at 1731750238762 (+417 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731750238779 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731750238779Flushing 1588230740/table: creating writer at 1731750239196 (+417 ms)Flushing 1588230740/table: appending metadata at 1731750239214 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731750239214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a766271: reopening flushed file at 1731750239236 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79f29704: reopening flushed file at 1731750239249 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b23c292: reopening flushed file at 1731750239259 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2044241f: reopening flushed file at 1731750239269 (+10 ms)Finished flush of dataSize ~25.47 KB/26078, heapSize ~41.58 KB/42576, currentSize=0 B/0 for 1588230740 in 1486ms, sequenceid=78, compaction requested=false at 1731750239286 (+17 ms)Writing region close event to WAL at 1731750239290 (+4 ms)Running coprocessor post-close hooks at 1731750239295 (+5 ms)Closed at 1731750239295
2024-11-16T09:43:59,296 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T09:43:59,397 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,36121,1731750222645; all regions closed.
2024-11-16T09:43:59,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741834_1010 (size=30662)
2024-11-16T09:43:59,407 DEBUG [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/oldWALs
2024-11-16T09:43:59,408 INFO  [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C36121%2C1731750222645.meta:.meta(num 1731750226493)
2024-11-16T09:43:59,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741832_1008 (size=4131)
2024-11-16T09:43:59,422 DEBUG [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/oldWALs
2024-11-16T09:43:59,422 INFO  [RS:0;037c58a19ddb:36121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C36121%2C1731750222645:(num 1731750225592)
2024-11-16T09:43:59,422 DEBUG [RS:0;037c58a19ddb:36121 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T09:43:59,422 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T09:43:59,423 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T09:43:59,423 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-16T09:43:59,423 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T09:43:59,424 INFO  [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T09:43:59,424 INFO  [RS:0;037c58a19ddb:36121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36121
2024-11-16T09:43:59,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,36121,1731750222645
2024-11-16T09:43:59,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T09:43:59,429 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T09:43:59,430 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,36121,1731750222645]
2024-11-16T09:43:59,432 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,36121,1731750222645 already deleted, retry=false
2024-11-16T09:43:59,432 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 037c58a19ddb,36121,1731750222645 expired; onlineServers=0
2024-11-16T09:43:59,432 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '037c58a19ddb,37567,1731750221953' *****
2024-11-16T09:43:59,432 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T09:43:59,432 INFO  [M:0;037c58a19ddb:37567 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T09:43:59,432 INFO  [M:0;037c58a19ddb:37567 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T09:43:59,433 DEBUG [M:0;037c58a19ddb:37567 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T09:43:59,433 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T09:43:59,433 DEBUG [M:0;037c58a19ddb:37567 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T09:43:59,433 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750225246 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750225246,5,FailOnTimeoutGroup]
2024-11-16T09:43:59,433 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750225248 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750225248,5,FailOnTimeoutGroup]
2024-11-16T09:43:59,433 INFO  [M:0;037c58a19ddb:37567 {}] hbase.ChoreService(370): Chore service for: master/037c58a19ddb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T09:43:59,433 INFO  [M:0;037c58a19ddb:37567 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T09:43:59,433 DEBUG [M:0;037c58a19ddb:37567 {}] master.HMaster(1795): Stopping service threads
2024-11-16T09:43:59,433 INFO  [M:0;037c58a19ddb:37567 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T09:43:59,433 INFO  [M:0;037c58a19ddb:37567 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T09:43:59,433 ERROR [M:0;037c58a19ddb:37567 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[HFileArchiver-3,5,PEWorkerGroup]
    Thread[HFileArchiver-4,5,PEWorkerGroup]
    Thread[HFileArchiver-5,5,PEWorkerGroup]
    Thread[HFileArchiver-6,5,PEWorkerGroup]
2024-11-16T09:43:59,433 INFO  [M:0;037c58a19ddb:37567 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T09:43:59,434 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T09:43:59,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T09:43:59,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T09:43:59,436 DEBUG [M:0;037c58a19ddb:37567 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-11-16T09:43:59,436 DEBUG [M:0;037c58a19ddb:37567 {}] master.ActiveMasterManager(353): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-11-16T09:43:59,437 INFO  [M:0;037c58a19ddb:37567 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/.lastflushedseqids
2024-11-16T09:43:59,449 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741848_1024 (size=72)
2024-11-16T09:43:59,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T09:43:59,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36121-0x101b076ca840001, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T09:43:59,532 INFO  [RS:0;037c58a19ddb:36121 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T09:43:59,533 INFO  [RS:0;037c58a19ddb:36121 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,36121,1731750222645; zookeeper connection closed.
2024-11-16T09:43:59,533 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e51d17d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e51d17d
2024-11-16T09:43:59,534 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T09:43:59,850 INFO  [M:0;037c58a19ddb:37567 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T09:43:59,850 INFO  [M:0;037c58a19ddb:37567 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T09:43:59,851 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T09:43:59,851 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:59,851 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:59,851 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T09:43:59,851 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:43:59,851 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=149.11 KB heapSize=178.89 KB
2024-11-16T09:43:59,871 DEBUG [M:0;037c58a19ddb:37567 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cf93c1cc22584a548871bcbd6cf3330d is 82, key is hbase:meta,,1/info:regioninfo/1731750226585/Put/seqid=0
2024-11-16T09:43:59,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741849_1025 (size=5672)
2024-11-16T09:43:59,876 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cf93c1cc22584a548871bcbd6cf3330d
2024-11-16T09:43:59,900 DEBUG [M:0;037c58a19ddb:37567 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1f0ed03f380140269950811f315bb28d is 1206, key is \x00\x00\x00\x00\x00\x00\x00\x16/proc:d/1731750231579/Put/seqid=0
2024-11-16T09:43:59,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741850_1026 (size=11519)
2024-11-16T09:44:00,307 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148.55 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1f0ed03f380140269950811f315bb28d
2024-11-16T09:44:00,334 DEBUG [M:0;037c58a19ddb:37567 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f393676b30ec4cd0b597a36480aa247e is 69, key is 037c58a19ddb,36121,1731750222645/rs:state/1731750225365/Put/seqid=0
2024-11-16T09:44:00,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741851_1027 (size=5156)
2024-11-16T09:44:00,739 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f393676b30ec4cd0b597a36480aa247e
2024-11-16T09:44:00,748 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cf93c1cc22584a548871bcbd6cf3330d as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cf93c1cc22584a548871bcbd6cf3330d
2024-11-16T09:44:00,755 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cf93c1cc22584a548871bcbd6cf3330d, entries=8, sequenceid=328, filesize=5.5 K
2024-11-16T09:44:00,756 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1f0ed03f380140269950811f315bb28d as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1f0ed03f380140269950811f315bb28d
2024-11-16T09:44:00,763 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1f0ed03f380140269950811f315bb28d, entries=39, sequenceid=328, filesize=11.2 K
2024-11-16T09:44:00,765 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f393676b30ec4cd0b597a36480aa247e as hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f393676b30ec4cd0b597a36480aa247e
2024-11-16T09:44:00,772 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32973/user/jenkins/test-data/77d466f7-f16b-eaf8-7bed-7fab14c8bb7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f393676b30ec4cd0b597a36480aa247e, entries=1, sequenceid=328, filesize=5.0 K
2024-11-16T09:44:00,773 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.11 KB/152686, heapSize ~178.59 KB/182880, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 922ms, sequenceid=328, compaction requested=false
2024-11-16T09:44:00,775 INFO  [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T09:44:00,775 DEBUG [M:0;037c58a19ddb:37567 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731750239851Disabling compacts and flushes for region at 1731750239851Disabling writes for close at 1731750239851Obtaining lock to block concurrent updates at 1731750239851Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731750239851Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=152686, getHeapSize=183120, getOffHeapSize=0, getCellsCount=389 at 1731750239852 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731750239852Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731750239853 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731750239870 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731750239870Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731750239883 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731750239900 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731750239900Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731750240314 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731750240333 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731750240333Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@473a017: reopening flushed file at 1731750240747 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1acb4c28: reopening flushed file at 1731750240755 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aafd3e6: reopening flushed file at 1731750240764 (+9 ms)Finished flush of dataSize ~149.11 KB/152686, heapSize ~178.59 KB/182880, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 922ms, sequenceid=328, compaction requested=false at 1731750240773 (+9 ms)Writing region close event to WAL at 1731750240775 (+2 ms)Closed at 1731750240775
2024-11-16T09:44:00,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36503 is added to blk_1073741830_1006 (size=174669)
2024-11-16T09:44:00,779 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T09:44:00,779 INFO  [M:0;037c58a19ddb:37567 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T09:44:00,779 INFO  [M:0;037c58a19ddb:37567 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37567
2024-11-16T09:44:00,779 INFO  [M:0;037c58a19ddb:37567 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T09:44:00,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T09:44:00,881 INFO  [M:0;037c58a19ddb:37567 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T09:44:00,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37567-0x101b076ca840000, quorum=127.0.0.1:53670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T09:44:00,887 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@156ffeaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T09:44:00,890 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T09:44:00,890 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T09:44:00,890 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T09:44:00,891 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/hadoop.log.dir/,STOPPED}
2024-11-16T09:44:00,894 WARN  [BP-1238687576-172.17.0.2-1731750218989 heartbeating to localhost/127.0.0.1:32973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T09:44:00,894 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T09:44:00,894 WARN  [BP-1238687576-172.17.0.2-1731750218989 heartbeating to localhost/127.0.0.1:32973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1238687576-172.17.0.2-1731750218989 (Datanode Uuid bce16ce0-27c0-4b48-a7a8-5ce93a5521b6) service to localhost/127.0.0.1:32973
2024-11-16T09:44:00,894 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T09:44:00,895 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/data/data1/current/BP-1238687576-172.17.0.2-1731750218989 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T09:44:00,895 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/cluster_d0054bcb-8860-1093-374b-1b7f299f9b8c/data/data2/current/BP-1238687576-172.17.0.2-1731750218989 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T09:44:00,896 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T09:44:00,904 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27b19a13{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T09:44:00,905 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T09:44:00,905 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T09:44:00,905 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T09:44:00,905 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1904c50-b570-e40d-2270-3339fa9929c5/hadoop.log.dir/,STOPPED}
2024-11-16T09:44:00,912 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T09:44:00,930 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down