2024-11-08 23:43:04,721 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4
2024-11-08 23:43:04,733 main DEBUG Took 0.009853 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-08 23:43:04,733 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-08 23:43:04,734 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-08 23:43:04,740 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-08 23:43:04,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,750 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-08 23:43:04,764 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,766 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,767 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,767 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,768 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,768 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,769 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,771 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,772 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,773 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,773 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,774 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,775 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,776 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,776 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,777 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,778 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,779 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 23:43:04,780 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,780 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-08 23:43:04,782 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 23:43:04,784 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-08 23:43:04,786 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-08 23:43:04,787 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-08 23:43:04,788 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-08 23:43:04,789 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-08 23:43:04,801 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-08 23:43:04,804 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-08 23:43:04,807 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-08 23:43:04,807 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-08 23:43:04,808 main DEBUG createAppenders(={Console})
2024-11-08 23:43:04,809 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 initialized
2024-11-08 23:43:04,809 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4
2024-11-08 23:43:04,809 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 OK.
2024-11-08 23:43:04,810 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-08 23:43:04,810 main DEBUG OutputStream closed
2024-11-08 23:43:04,810 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-08 23:43:04,810 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-08 23:43:04,810 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4803b726 OK
2024-11-08 23:43:04,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-08 23:43:04,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-08 23:43:04,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-08 23:43:04,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-08 23:43:04,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-08 23:43:04,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-08 23:43:04,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-08 23:43:04,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-08 23:43:04,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-08 23:43:04,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-08 23:43:04,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-08 23:43:04,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-08 23:43:04,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-08 23:43:04,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-08 23:43:04,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-08 23:43:04,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-08 23:43:04,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-08 23:43:04,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-08 23:43:04,894 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-08 23:43:04,894 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null
2024-11-08 23:43:04,895 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-08 23:43:04,895 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK.
2024-11-08T23:43:05,136 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009
2024-11-08 23:43:05,138 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-08 23:43:05,139 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-08T23:43:05,148 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure timeout: 13 mins
2024-11-08T23:43:05,165 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-08T23:43:05,179 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08, deleteOnExit=true
2024-11-08T23:43:05,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-08T23:43:05,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/test.cache.data in system properties and HBase conf
2024-11-08T23:43:05,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/hadoop.tmp.dir in system properties and HBase conf
2024-11-08T23:43:05,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/hadoop.log.dir in system properties and HBase conf
2024-11-08T23:43:05,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-08T23:43:05,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-08T23:43:05,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-08T23:43:05,270 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-08T23:43:05,358 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-08T23:43:05,361 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-08T23:43:05,362 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-08T23:43:05,362 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-08T23:43:05,362 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T23:43:05,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-08T23:43:05,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-08T23:43:05,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T23:43:05,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T23:43:05,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-08T23:43:05,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/nfs.dump.dir in system properties and HBase conf
2024-11-08T23:43:05,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/java.io.tmpdir in system properties and HBase conf
2024-11-08T23:43:05,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T23:43:05,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-08T23:43:05,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-08T23:43:06,160 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-08T23:43:06,233 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2219ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-08T23:43:06,306 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T23:43:06,371 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T23:43:06,391 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T23:43:06,392 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T23:43:06,393 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T23:43:06,405 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T23:43:06,408 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a627e45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/hadoop.log.dir/,AVAILABLE}
2024-11-08T23:43:06,409 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f35049c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T23:43:06,596 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34d8ada7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/java.io.tmpdir/jetty-localhost-43607-hadoop-hdfs-3_4_1-tests_jar-_-any-4707231980855089504/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T23:43:06,603 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6067377f{HTTP/1.1, (http/1.1)}{localhost:43607}
2024-11-08T23:43:06,603 INFO  [Time-limited test {}] server.Server(415): Started @2589ms
2024-11-08T23:43:06,993 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T23:43:07,002 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T23:43:07,003 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T23:43:07,003 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T23:43:07,004 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T23:43:07,005 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f4f11e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/hadoop.log.dir/,AVAILABLE}
2024-11-08T23:43:07,005 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@464c0fed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T23:43:07,126 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18508316{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/java.io.tmpdir/jetty-localhost-45869-hadoop-hdfs-3_4_1-tests_jar-_-any-8654942544469490905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T23:43:07,126 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@d3378eb{HTTP/1.1, (http/1.1)}{localhost:45869}
2024-11-08T23:43:07,127 INFO  [Time-limited test {}] server.Server(415): Started @3113ms
2024-11-08T23:43:07,183 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T23:43:07,658 WARN  [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/data/data1/current/BP-1559241834-172.17.0.2-1731109385919/current, will proceed with Du for space computation calculation, 
2024-11-08T23:43:07,658 WARN  [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/data/data2/current/BP-1559241834-172.17.0.2-1731109385919/current, will proceed with Du for space computation calculation, 
2024-11-08T23:43:07,689 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T23:43:07,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x608c08eab0055f0a with lease ID 0x1fee01ee6e84e2dc: Processing first storage report for DS-cb95599f-c7e2-4028-aab8-3c5772346ae7 from datanode DatanodeRegistration(127.0.0.1:33713, datanodeUuid=88a7e4e5-1681-48b0-866e-41fa7dc5e539, infoPort=44937, infoSecurePort=0, ipcPort=36149, storageInfo=lv=-57;cid=testClusterID;nsid=2090207189;c=1731109385919)
2024-11-08T23:43:07,744 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x608c08eab0055f0a with lease ID 0x1fee01ee6e84e2dc: from storage DS-cb95599f-c7e2-4028-aab8-3c5772346ae7 node DatanodeRegistration(127.0.0.1:33713, datanodeUuid=88a7e4e5-1681-48b0-866e-41fa7dc5e539, infoPort=44937, infoSecurePort=0, ipcPort=36149, storageInfo=lv=-57;cid=testClusterID;nsid=2090207189;c=1731109385919), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-08T23:43:07,745 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x608c08eab0055f0a with lease ID 0x1fee01ee6e84e2dc: Processing first storage report for DS-543d60a0-f5cb-470e-99fe-093433e4847a from datanode DatanodeRegistration(127.0.0.1:33713, datanodeUuid=88a7e4e5-1681-48b0-866e-41fa7dc5e539, infoPort=44937, infoSecurePort=0, ipcPort=36149, storageInfo=lv=-57;cid=testClusterID;nsid=2090207189;c=1731109385919)
2024-11-08T23:43:07,745 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x608c08eab0055f0a with lease ID 0x1fee01ee6e84e2dc: from storage DS-543d60a0-f5cb-470e-99fe-093433e4847a node DatanodeRegistration(127.0.0.1:33713, datanodeUuid=88a7e4e5-1681-48b0-866e-41fa7dc5e539, infoPort=44937, infoSecurePort=0, ipcPort=36149, storageInfo=lv=-57;cid=testClusterID;nsid=2090207189;c=1731109385919), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T23:43:07,763 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009
2024-11-08T23:43:07,841 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/zookeeper_0, clientPort=64298, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-08T23:43:07,850 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64298
2024-11-08T23:43:07,865 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:07,869 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:08,106 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741825_1001 (size=7)
2024-11-08T23:43:08,514 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1 with version=8
2024-11-08T23:43:08,514 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/hbase-staging
2024-11-08T23:43:08,612 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-08T23:43:08,868 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a15bf3ddec7f:0 server-side Connection retries=45
2024-11-08T23:43:08,878 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T23:43:08,879 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T23:43:08,883 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T23:43:08,884 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T23:43:08,884 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T23:43:09,018 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-08T23:43:09,079 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-08T23:43:09,087 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-08T23:43:09,091 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T23:43:09,118 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4878 (auto-detected)
2024-11-08T23:43:09,119 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-08T23:43:09,137 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46285
2024-11-08T23:43:09,158 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46285 connecting to ZooKeeper ensemble=127.0.0.1:64298
2024-11-08T23:43:09,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462850x0, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T23:43:09,191 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46285-0x100f9f6e8bb0000 connected
2024-11-08T23:43:09,216 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:09,219 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:09,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T23:43:09,233 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1, hbase.cluster.distributed=false
2024-11-08T23:43:09,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T23:43:09,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46285
2024-11-08T23:43:09,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46285
2024-11-08T23:43:09,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46285
2024-11-08T23:43:09,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46285
2024-11-08T23:43:09,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46285
2024-11-08T23:43:09,383 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a15bf3ddec7f:0 server-side Connection retries=45
2024-11-08T23:43:09,384 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T23:43:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T23:43:09,385 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T23:43:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T23:43:09,385 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T23:43:09,388 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T23:43:09,390 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T23:43:09,391 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40483
2024-11-08T23:43:09,393 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40483 connecting to ZooKeeper ensemble=127.0.0.1:64298
2024-11-08T23:43:09,394 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:09,397 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:09,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404830x0, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T23:43:09,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404830x0, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T23:43:09,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40483-0x100f9f6e8bb0001 connected
2024-11-08T23:43:09,407 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T23:43:09,416 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T23:43:09,419 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T23:43:09,424 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T23:43:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40483
2024-11-08T23:43:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40483
2024-11-08T23:43:09,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40483
2024-11-08T23:43:09,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40483
2024-11-08T23:43:09,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40483
2024-11-08T23:43:09,442 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a15bf3ddec7f:46285
2024-11-08T23:43:09,443 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:09,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T23:43:09,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T23:43:09,452 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:09,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T23:43:09,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:09,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:09,474 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-08T23:43:09,475 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a15bf3ddec7f,46285,1731109388673 from backup master directory
2024-11-08T23:43:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T23:43:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T23:43:09,480 WARN  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T23:43:09,480 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:09,483 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-08T23:43:09,484 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-08T23:43:09,539 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/hbase.id] with ID: 3cab4b9f-d7ae-4021-b975-a02a3194830f
2024-11-08T23:43:09,540 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/.tmp/hbase.id
2024-11-08T23:43:09,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741826_1002 (size=42)
2024-11-08T23:43:09,952 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/.tmp/hbase.id]:[hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/hbase.id]
2024-11-08T23:43:09,994 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T23:43:09,999 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-08T23:43:10,018 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-08T23:43:10,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:10,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:10,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741827_1003 (size=196)
2024-11-08T23:43:10,453 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-08T23:43:10,455 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-08T23:43:10,471 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:10,475 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T23:43:10,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741828_1004 (size=1189)
2024-11-08T23:43:10,924 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store
2024-11-08T23:43:10,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741829_1005 (size=34)
2024-11-08T23:43:11,349 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-08T23:43:11,352 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:11,353 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-08T23:43:11,353 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:11,354 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:11,355 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-08T23:43:11,355 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:11,355 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:11,356 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731109391353Disabling compacts and flushes for region at 1731109391353Disabling writes for close at 1731109391355 (+2 ms)Writing region close event to WAL at 1731109391355Closed at 1731109391355
2024-11-08T23:43:11,358 WARN  [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/.initializing
2024-11-08T23:43:11,359 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/WALs/a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:11,367 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-08T23:43:11,381 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a15bf3ddec7f%2C46285%2C1731109388673, suffix=, logDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/WALs/a15bf3ddec7f,46285,1731109388673, archiveDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/oldWALs, maxLogs=10
2024-11-08T23:43:11,403 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/WALs/a15bf3ddec7f,46285,1731109388673/a15bf3ddec7f%2C46285%2C1731109388673.1731109391385, exclude list is [], retry=0
2024-11-08T23:43:11,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33713,DS-cb95599f-c7e2-4028-aab8-3c5772346ae7,DISK]
2024-11-08T23:43:11,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf.
2024-11-08T23:43:11,458 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/WALs/a15bf3ddec7f,46285,1731109388673/a15bf3ddec7f%2C46285%2C1731109388673.1731109391385
2024-11-08T23:43:11,459 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937)]
2024-11-08T23:43:11,459 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-08T23:43:11,460 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:11,463 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,464 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,500 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T23:43:11,527 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-08T23:43:11,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:11,534 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:11,534 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T23:43:11,538 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-08T23:43:11,538 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:11,539 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:11,539 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T23:43:11,541 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-08T23:43:11,542 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:11,542 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:11,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T23:43:11,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-08T23:43:11,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:11,546 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:11,547 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,550 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,551 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,556 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,557 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,560 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T23:43:11,563 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T23:43:11,567 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:11,568 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70651029, jitterRate=0.05278237164020538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T23:43:11,574 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731109391475Initializing all the Stores at 1731109391478 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109391478Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109391479 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109391479Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109391479Cleaning up temporary data from old regions at 1731109391557 (+78 ms)Region opened successfully at 1731109391574 (+17 ms)
2024-11-08T23:43:11,575 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-08T23:43:11,608 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10dd9b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0
2024-11-08T23:43:11,641 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-08T23:43:11,653 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:11,653 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:11,657 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:11,658 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:11,663 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T23:43:11,663 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-08T23:43:11,689 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-08T23:43:11,697 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-08T23:43:11,699 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-08T23:43:11,701 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-08T23:43:11,702 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-08T23:43:11,704 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-08T23:43:11,706 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-08T23:43:11,710 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-08T23:43:11,712 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-08T23:43:11,713 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-08T23:43:11,715 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-08T23:43:11,734 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-08T23:43:11,735 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-08T23:43:11,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T23:43:11,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T23:43:11,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:11,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:11,742 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a15bf3ddec7f,46285,1731109388673, sessionid=0x100f9f6e8bb0000, setting cluster-up flag (Was=false)
2024-11-08T23:43:11,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:11,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:11,763 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-08T23:43:11,765 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:11,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:11,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:11,778 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-08T23:43:11,779 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:11,786 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-08T23:43:11,832 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(746): ClusterId : 3cab4b9f-d7ae-4021-b975-a02a3194830f
2024-11-08T23:43:11,835 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-08T23:43:11,840 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-08T23:43:11,840 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-08T23:43:11,843 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-08T23:43:11,844 DEBUG [RS:0;a15bf3ddec7f:40483 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc37885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a15bf3ddec7f/172.17.0.2:0
2024-11-08T23:43:11,858 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a15bf3ddec7f:40483
2024-11-08T23:43:11,861 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-08T23:43:11,861 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-08T23:43:11,861 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-08T23:43:11,861 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:11,864 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,46285,1731109388673 with port=40483, startcode=1731109389343
2024-11-08T23:43:11,871 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-08T23:43:11,875 DEBUG [RS:0;a15bf3ddec7f:40483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-08T23:43:11,879 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-08T23:43:11,887 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a15bf3ddec7f,46285,1731109388673
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-08T23:43:11,895 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5
2024-11-08T23:43:11,895 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5
2024-11-08T23:43:11,896 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5
2024-11-08T23:43:11,896 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=5, maxPoolSize=5
2024-11-08T23:43:11,896 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a15bf3ddec7f:0, corePoolSize=10, maxPoolSize=10
2024-11-08T23:43:11,896 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:11,897 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2
2024-11-08T23:43:11,897 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:11,901 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109421901
2024-11-08T23:43:11,903 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-08T23:43:11,904 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-08T23:43:11,904 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:11,905 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-08T23:43:11,910 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-08T23:43:11,910 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-08T23:43:11,911 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-08T23:43:11,911 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51741, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-08T23:43:11,911 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-08T23:43:11,912 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:11,912 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-08T23:43:11,912 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:11,916 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-08T23:43:11,918 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-08T23:43:11,918 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-08T23:43:11,918 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:11,924 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-08T23:43:11,925 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-08T23:43:11,927 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109391926,5,FailOnTimeoutGroup]
2024-11-08T23:43:11,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741831_1007 (size=1321)
2024-11-08T23:43:11,927 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109391927,5,FailOnTimeoutGroup]
2024-11-08T23:43:11,928 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:11,928 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-08T23:43:11,929 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:11,929 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:11,948 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-11-08T23:43:11,948 WARN  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-11-08T23:43:12,049 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(2659): reportForDuty to master=a15bf3ddec7f,46285,1731109388673 with port=40483, startcode=1731109389343
2024-11-08T23:43:12,051 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,054 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] master.ServerManager(517): Registering regionserver=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,062 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:12,062 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38449
2024-11-08T23:43:12,062 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-08T23:43:12,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T23:43:12,068 DEBUG [RS:0;a15bf3ddec7f:40483 {}] zookeeper.ZKUtil(111): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,069 WARN  [RS:0;a15bf3ddec7f:40483 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T23:43:12,069 INFO  [RS:0;a15bf3ddec7f:40483 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T23:43:12,069 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,070 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a15bf3ddec7f,40483,1731109389343]
2024-11-08T23:43:12,095 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-08T23:43:12,109 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-08T23:43:12,113 INFO  [RS:0;a15bf3ddec7f:40483 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-08T23:43:12,114 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,115 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-08T23:43:12,120 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-08T23:43:12,121 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,122 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,122 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,122 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,122 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,123 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,123 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=2, maxPoolSize=2
2024-11-08T23:43:12,123 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,124 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,124 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,124 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,124 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,125 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a15bf3ddec7f:0, corePoolSize=1, maxPoolSize=1
2024-11-08T23:43:12,125 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3
2024-11-08T23:43:12,125 DEBUG [RS:0;a15bf3ddec7f:40483 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a15bf3ddec7f:0, corePoolSize=3, maxPoolSize=3
2024-11-08T23:43:12,126 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,126 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,127 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,127 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,127 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,127 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,40483,1731109389343-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T23:43:12,146 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-08T23:43:12,148 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,40483,1731109389343-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,148 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,148 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.Replication(171): a15bf3ddec7f,40483,1731109389343 started
2024-11-08T23:43:12,167 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:12,168 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1482): Serving as a15bf3ddec7f,40483,1731109389343, RpcServer on a15bf3ddec7f/172.17.0.2:40483, sessionid=0x100f9f6e8bb0001
2024-11-08T23:43:12,169 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-08T23:43:12,169 DEBUG [RS:0;a15bf3ddec7f:40483 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,169 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,40483,1731109389343'
2024-11-08T23:43:12,169 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-08T23:43:12,170 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-08T23:43:12,171 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-08T23:43:12,171 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-08T23:43:12,171 DEBUG [RS:0;a15bf3ddec7f:40483 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,171 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a15bf3ddec7f,40483,1731109389343'
2024-11-08T23:43:12,172 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-08T23:43:12,172 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-08T23:43:12,173 DEBUG [RS:0;a15bf3ddec7f:40483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-08T23:43:12,173 INFO  [RS:0;a15bf3ddec7f:40483 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-08T23:43:12,173 INFO  [RS:0;a15bf3ddec7f:40483 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-08T23:43:12,279 INFO  [RS:0;a15bf3ddec7f:40483 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-11-08T23:43:12,283 INFO  [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a15bf3ddec7f%2C40483%2C1731109389343, suffix=, logDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343, archiveDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/oldWALs, maxLogs=32
2024-11-08T23:43:12,300 DEBUG [RS:0;a15bf3ddec7f:40483 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343/a15bf3ddec7f%2C40483%2C1731109389343.1731109392285, exclude list is [], retry=0
2024-11-08T23:43:12,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33713,DS-cb95599f-c7e2-4028-aab8-3c5772346ae7,DISK]
2024-11-08T23:43:12,308 INFO  [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343/a15bf3ddec7f%2C40483%2C1731109389343.1731109392285
2024-11-08T23:43:12,309 DEBUG [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937)]
2024-11-08T23:43:12,330 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-08T23:43:12,330 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:12,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741833_1009 (size=32)
2024-11-08T23:43:12,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:12,744 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-08T23:43:12,747 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-08T23:43:12,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:12,748 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:12,748 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-08T23:43:12,751 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-08T23:43:12,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:12,752 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:12,752 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-08T23:43:12,754 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-08T23:43:12,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:12,755 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:12,755 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-08T23:43:12,758 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-08T23:43:12,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:12,759 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:12,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-08T23:43:12,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740
2024-11-08T23:43:12,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740
2024-11-08T23:43:12,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-08T23:43:12,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-08T23:43:12,765 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T23:43:12,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-08T23:43:12,771 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:12,772 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72974588, jitterRate=0.08740609884262085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T23:43:12,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731109392742Initializing all the Stores at 1731109392744 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109392744Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109392744Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109392744Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109392744Cleaning up temporary data from old regions at 1731109392764 (+20 ms)Region opened successfully at 1731109392774 (+10 ms)
2024-11-08T23:43:12,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-08T23:43:12,774 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-08T23:43:12,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-08T23:43:12,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-08T23:43:12,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-08T23:43:12,776 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-08T23:43:12,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731109392774Disabling compacts and flushes for region at 1731109392774Disabling writes for close at 1731109392774Writing region close event to WAL at 1731109392775 (+1 ms)Closed at 1731109392776 (+1 ms)
2024-11-08T23:43:12,779 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:12,779 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-08T23:43:12,786 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-08T23:43:12,797 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-08T23:43:12,799 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-08T23:43:12,952 DEBUG [a15bf3ddec7f:46285 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-08T23:43:12,964 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:12,971 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPENING
2024-11-08T23:43:12,976 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-08T23:43:12,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:12,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:12,978 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:12,978 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:12,980 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-08T23:43:12,982 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:13,157 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-08T23:43:13,160 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55717, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-08T23:43:13,170 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-08T23:43:13,171 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T23:43:13,171 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-11-08T23:43:13,174 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a15bf3ddec7f%2C40483%2C1731109389343.meta, suffix=.meta, logDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343, archiveDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/oldWALs, maxLogs=32
2024-11-08T23:43:13,191 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343/a15bf3ddec7f%2C40483%2C1731109389343.meta.1731109393176.meta, exclude list is [], retry=0
2024-11-08T23:43:13,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33713,DS-cb95599f-c7e2-4028-aab8-3c5772346ae7,DISK]
2024-11-08T23:43:13,197 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/WALs/a15bf3ddec7f,40483,1731109389343/a15bf3ddec7f%2C40483%2C1731109389343.meta.1731109393176.meta
2024-11-08T23:43:13,198 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937)]
2024-11-08T23:43:13,198 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-08T23:43:13,200 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-08T23:43:13,203 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-08T23:43:13,207 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-08T23:43:13,211 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-08T23:43:13,212 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:13,212 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-08T23:43:13,212 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-08T23:43:13,216 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-08T23:43:13,217 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-08T23:43:13,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:13,218 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:13,218 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-08T23:43:13,220 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-08T23:43:13,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:13,221 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:13,221 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-08T23:43:13,222 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-08T23:43:13,222 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:13,223 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:13,223 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-08T23:43:13,225 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-08T23:43:13,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:13,226 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T23:43:13,226 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-08T23:43:13,227 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740
2024-11-08T23:43:13,230 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740
2024-11-08T23:43:13,232 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-08T23:43:13,232 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-08T23:43:13,233 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T23:43:13,236 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-08T23:43:13,238 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63274469, jitterRate=-0.057136937975883484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T23:43:13,238 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-08T23:43:13,240 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731109393213Writing region info on filesystem at 1731109393213Initializing all the Stores at 1731109393215 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109393215Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109393215Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109393215Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731109393216 (+1 ms)Cleaning up temporary data from old regions at 1731109393232 (+16 ms)Running coprocessor post-open hooks at 1731109393238 (+6 ms)Region opened successfully at 1731109393240 (+2 ms)
2024-11-08T23:43:13,247 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731109393148
2024-11-08T23:43:13,259 DEBUG [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-08T23:43:13,260 INFO  [RS_OPEN_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-08T23:43:13,261 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:13,263 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:13,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:13,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:13,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:13,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:13,270 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:13,274 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-08T23:43:13,275 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a15bf3ddec7f,40483,1731109389343 in 289 msec
2024-11-08T23:43:13,281 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-08T23:43:13,281 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 491 msec
2024-11-08T23:43:13,282 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:13,282 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-08T23:43:13,303 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-08T23:43:13,305 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,40483,1731109389343, seqNum=-1]
2024-11-08T23:43:13,327 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T23:43:13,330 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36909, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T23:43:13,351 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5350 sec
2024-11-08T23:43:13,352 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731109393352, completionTime=-1
2024-11-08T23:43:13,354 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-08T23:43:13,355 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:13,382 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:13,382 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109453382
2024-11-08T23:43:13,383 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109513383
2024-11-08T23:43:13,383 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec
2024-11-08T23:43:13,385 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46285,1731109388673-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,385 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46285,1731109388673-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,385 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46285,1731109388673-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,387 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a15bf3ddec7f:46285, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,388 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,388 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,393 DEBUG [master/a15bf3ddec7f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-08T23:43:13,414 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.934sec
2024-11-08T23:43:13,415 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-08T23:43:13,417 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-08T23:43:13,418 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-08T23:43:13,418 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-08T23:43:13,418 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-08T23:43:13,419 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46285,1731109388673-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T23:43:13,420 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46285,1731109388673-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-08T23:43:13,427 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-08T23:43:13,428 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-08T23:43:13,429 INFO  [master/a15bf3ddec7f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a15bf3ddec7f,46285,1731109388673-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T23:43:13,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5677489c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T23:43:13,444 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-08T23:43:13,444 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-08T23:43:13,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a15bf3ddec7f,46285,-1 for getting cluster id
2024-11-08T23:43:13,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-08T23:43:13,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cab4b9f-d7ae-4021-b975-a02a3194830f'
2024-11-08T23:43:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-08T23:43:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cab4b9f-d7ae-4021-b975-a02a3194830f"

2024-11-08T23:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2479a655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T23:43:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a15bf3ddec7f,46285,-1]
2024-11-08T23:43:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-08T23:43:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T23:43:13,469 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-08T23:43:13,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3caf25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T23:43:13,472 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-08T23:43:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a15bf3ddec7f,40483,1731109389343, seqNum=-1]
2024-11-08T23:43:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T23:43:13,481 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T23:43:13,521 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:13,534 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTable Thread=207, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=3253
2024-11-08T23:43:13,538 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T23:43:13,550 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:13,553 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T23:43:13,557 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:13,560 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T23:43:13,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741835_1011 (size=639)
2024-11-08T23:43:13,975 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 173857b1f7ac4173a9669368af15cf84, NAME => 'testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:13,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741836_1012 (size=51)
2024-11-08T23:43:14,385 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:14,385 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1722): Closing 173857b1f7ac4173a9669368af15cf84, disabling compactions & flushes
2024-11-08T23:43:14,385 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,385 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,385 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84. after waiting 0 ms
2024-11-08T23:43:14,385 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,385 INFO  [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,385 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 173857b1f7ac4173a9669368af15cf84:
Waiting for close lock at 1731109394385Disabling compacts and flushes for region at 1731109394385Disabling writes for close at 1731109394385Writing region close event to WAL at 1731109394385Closed at 1731109394385
2024-11-08T23:43:14,388 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T23:43:14,393 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731109394388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109394388"}]},"ts":"1731109394388"}
2024-11-08T23:43:14,399 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-08T23:43:14,400 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T23:43:14,403 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109394400"}]},"ts":"1731109394400"}
2024-11-08T23:43:14,407 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLING in hbase:meta
2024-11-08T23:43:14,409 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, ASSIGN}]
2024-11-08T23:43:14,412 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, ASSIGN
2024-11-08T23:43:14,413 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,40483,1731109389343; forceNewPlan=false, retain=false
2024-11-08T23:43:14,565 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=173857b1f7ac4173a9669368af15cf84, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:14,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, ASSIGN because future has completed
2024-11-08T23:43:14,570 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 173857b1f7ac4173a9669368af15cf84, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:14,730 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,730 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 173857b1f7ac4173a9669368af15cf84, NAME => 'testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.', STARTKEY => '', ENDKEY => ''}
2024-11-08T23:43:14,731 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTable 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,731 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:14,731 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,731 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,734 INFO  [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 173857b1f7ac4173a9669368af15cf84 
2024-11-08T23:43:14,736 INFO  [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 173857b1f7ac4173a9669368af15cf84 columnFamilyName f1
2024-11-08T23:43:14,736 DEBUG [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:14,737 INFO  [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] regionserver.HStore(327): Store=173857b1f7ac4173a9669368af15cf84/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:14,737 INFO  [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 173857b1f7ac4173a9669368af15cf84 
2024-11-08T23:43:14,739 INFO  [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 173857b1f7ac4173a9669368af15cf84 columnFamilyName f2
2024-11-08T23:43:14,739 DEBUG [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:14,740 INFO  [StoreOpener-173857b1f7ac4173a9669368af15cf84-1 {}] regionserver.HStore(327): Store=173857b1f7ac4173a9669368af15cf84/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:14,740 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,742 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,742 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,744 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,744 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,745 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTable descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T23:43:14,747 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,750 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:14,751 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 173857b1f7ac4173a9669368af15cf84; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59097013, jitterRate=-0.11938588321208954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T23:43:14,751 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,752 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 173857b1f7ac4173a9669368af15cf84:
Running coprocessor pre-open hook at 1731109394731Writing region info on filesystem at 1731109394731Initializing all the Stores at 1731109394733 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109394733Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109394733Cleaning up temporary data from old regions at 1731109394744 (+11 ms)Running coprocessor post-open hooks at 1731109394752 (+8 ms)Region opened successfully at 1731109394752
2024-11-08T23:43:14,755 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84., pid=6, masterSystemTime=1731109394723
2024-11-08T23:43:14,758 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,758 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,759 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=173857b1f7ac4173a9669368af15cf84, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:14,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 173857b1f7ac4173a9669368af15cf84, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:14,768 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-08T23:43:14,768 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 173857b1f7ac4173a9669368af15cf84, server=a15bf3ddec7f,40483,1731109389343 in 194 msec
2024-11-08T23:43:14,772 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-08T23:43:14,772 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, ASSIGN in 359 msec
2024-11-08T23:43:14,773 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T23:43:14,773 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109394773"}]},"ts":"1731109394773"}
2024-11-08T23:43:14,776 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLED in hbase:meta
2024-11-08T23:43:14,777 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T23:43:14,780 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable in 1.2330 sec
2024-11-08T23:43:14,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:14,811 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109394811"}]},"ts":"1731109394811"}
2024-11-08T23:43:14,814 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLING in hbase:meta
2024-11-08T23:43:14,814 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTable to state=DISABLING
2024-11-08T23:43:14,816 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTable}]
2024-11-08T23:43:14,823 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, UNASSIGN}]
2024-11-08T23:43:14,825 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, UNASSIGN
2024-11-08T23:43:14,827 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=173857b1f7ac4173a9669368af15cf84, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:14,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, UNASSIGN because future has completed
2024-11-08T23:43:14,831 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:14,831 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 173857b1f7ac4173a9669368af15cf84, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:14,990 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:14,990 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T23:43:14,990 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 173857b1f7ac4173a9669368af15cf84, disabling compactions & flushes
2024-11-08T23:43:14,991 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,991 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,991 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84. after waiting 0 ms
2024-11-08T23:43:14,991 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:14,997 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T23:43:15,000 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.
2024-11-08T23:43:15,000 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 173857b1f7ac4173a9669368af15cf84:
Waiting for close lock at 1731109394990Running coprocessor pre-close hooks at 1731109394990Disabling compacts and flushes for region at 1731109394990Disabling writes for close at 1731109394991 (+1 ms)Writing region close event to WAL at 1731109394991Running coprocessor post-close hooks at 1731109394998 (+7 ms)Closed at 1731109395000 (+2 ms)
2024-11-08T23:43:15,004 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:15,005 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=173857b1f7ac4173a9669368af15cf84, regionState=CLOSED
2024-11-08T23:43:15,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 173857b1f7ac4173a9669368af15cf84, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:15,013 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-08T23:43:15,013 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 173857b1f7ac4173a9669368af15cf84, server=a15bf3ddec7f,40483,1731109389343 in 178 msec
2024-11-08T23:43:15,016 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-08T23:43:15,016 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=173857b1f7ac4173a9669368af15cf84, UNASSIGN in 191 msec
2024-11-08T23:43:15,020 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-08T23:43:15,020 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTable in 201 msec
2024-11-08T23:43:15,022 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109395022"}]},"ts":"1731109395022"}
2024-11-08T23:43:15,025 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLED in hbase:meta
2024-11-08T23:43:15,025 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTable to state=DISABLED
2024-11-08T23:43:15,029 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable in 224 msec
2024-11-08T23:43:15,057 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T23:43:15,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-08T23:43:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c1af104
2024-11-08T23:43:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-08T23:43:15,072 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-08T23:43:15,075 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTable
2024-11-08T23:43:15,079 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTable
2024-11-08T23:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] ipc.CallRunner(138): callId: 2 service: MasterService methodName: DisableTable size: 79 connection: 172.17.0.2:37578 deadline: 1731109455078, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
2024-11-08T23:43:15,085 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTable failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-08T23:43:15,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTable already disabled, so just deleting it.
2024-11-08T23:43:15,088 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTable
2024-11-08T23:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] procedure2.ProcedureExecutor(1139): Stored pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:15,097 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:15,099 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=12, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12
2024-11-08T23:43:15,107 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:15,112 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/f1, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/f2, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/recovered.edits]
2024-11-08T23:43:15,125 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/recovered.edits/4.seqid to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/archive/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84/recovered.edits/4.seqid
2024-11-08T23:43:15,126 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTable/173857b1f7ac4173a9669368af15cf84
2024-11-08T23:43:15,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTable regions
2024-11-08T23:43:15,132 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=12, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:15,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40483 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms
2024-11-08T23:43:15,142 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTable from hbase:meta
2024-11-08T23:43:15,173 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTable' descriptor.
2024-11-08T23:43:15,175 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=12, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:15,175 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTable' from region states.
2024-11-08T23:43:15,176 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731109395175"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:15,180 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META
2024-11-08T23:43:15,180 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 173857b1f7ac4173a9669368af15cf84, NAME => 'testDisableTable,,1731109393543.173857b1f7ac4173a9669368af15cf84.', STARTKEY => '', ENDKEY => ''}]
2024-11-08T23:43:15,180 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTable' as deleted.
2024-11-08T23:43:15,180 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731109395180"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:15,183 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTable state from META
2024-11-08T23:43:15,184 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=12, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:15,187 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable in 94 msec
2024-11-08T23:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12
2024-11-08T23:43:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTable
2024-11-08T23:43:15,217 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTable completed
2024-11-08T23:43:15,228 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTable Thread=217 (was 207)
Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1825474281_22 at /127.0.0.1:54754 [Waiting for operation #6]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RSProcedureDispatcher-pool-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=119 (was 119), ProcessCount=11 (was 11), AvailableMemoryMB=3236 (was 3253)
2024-11-08T23:43:15,236 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=217, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=3235
2024-11-08T23:43:15,237 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T23:43:15,239 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:15,240 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T23:43:15,241 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:15,242 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T23:43:15,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741837_1013 (size=652)
2024-11-08T23:43:15,654 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7e033ff00998e1082f711e70d2d1335b, NAME => 'testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTableMultipleTimes', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:15,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741838_1014 (size=64)
2024-11-08T23:43:16,064 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:16,064 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1722): Closing 7e033ff00998e1082f711e70d2d1335b, disabling compactions & flushes
2024-11-08T23:43:16,064 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,064 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,064 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b. after waiting 0 ms
2024-11-08T23:43:16,064 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,064 INFO  [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,064 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7e033ff00998e1082f711e70d2d1335b:
Waiting for close lock at 1731109396064Disabling compacts and flushes for region at 1731109396064Disabling writes for close at 1731109396064Writing region close event to WAL at 1731109396064Closed at 1731109396064
2024-11-08T23:43:16,066 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T23:43:16,066 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731109396066"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109396066"}]},"ts":"1731109396066"}
2024-11-08T23:43:16,069 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-08T23:43:16,071 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T23:43:16,071 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109396071"}]},"ts":"1731109396071"}
2024-11-08T23:43:16,073 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLING in hbase:meta
2024-11-08T23:43:16,074 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, ASSIGN}]
2024-11-08T23:43:16,076 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, ASSIGN
2024-11-08T23:43:16,077 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,40483,1731109389343; forceNewPlan=false, retain=false
2024-11-08T23:43:16,228 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7e033ff00998e1082f711e70d2d1335b, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:16,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, ASSIGN because future has completed
2024-11-08T23:43:16,233 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e033ff00998e1082f711e70d2d1335b, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:16,391 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,391 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e033ff00998e1082f711e70d2d1335b, NAME => 'testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.', STARTKEY => '', ENDKEY => ''}
2024-11-08T23:43:16,391 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTableMultipleTimes 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,391 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:16,392 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,392 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,394 INFO  [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 7e033ff00998e1082f711e70d2d1335b 
2024-11-08T23:43:16,396 INFO  [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e033ff00998e1082f711e70d2d1335b columnFamilyName f1
2024-11-08T23:43:16,396 DEBUG [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:16,396 INFO  [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] regionserver.HStore(327): Store=7e033ff00998e1082f711e70d2d1335b/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:16,397 INFO  [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 7e033ff00998e1082f711e70d2d1335b 
2024-11-08T23:43:16,399 INFO  [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e033ff00998e1082f711e70d2d1335b columnFamilyName f2
2024-11-08T23:43:16,400 DEBUG [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:16,400 INFO  [StoreOpener-7e033ff00998e1082f711e70d2d1335b-1 {}] regionserver.HStore(327): Store=7e033ff00998e1082f711e70d2d1335b/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:16,400 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,401 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,402 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,403 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,403 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,404 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTableMultipleTimes descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T23:43:16,406 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,409 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:16,410 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 7e033ff00998e1082f711e70d2d1335b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71406366, jitterRate=0.06403777003288269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T23:43:16,410 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,411 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 7e033ff00998e1082f711e70d2d1335b:
Running coprocessor pre-open hook at 1731109396392Writing region info on filesystem at 1731109396392Initializing all the Stores at 1731109396393 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109396393Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109396394 (+1 ms)Cleaning up temporary data from old regions at 1731109396403 (+9 ms)Running coprocessor post-open hooks at 1731109396410 (+7 ms)Region opened successfully at 1731109396411 (+1 ms)
2024-11-08T23:43:16,412 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b., pid=15, masterSystemTime=1731109396386
2024-11-08T23:43:16,415 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,415 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,416 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7e033ff00998e1082f711e70d2d1335b, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:16,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e033ff00998e1082f711e70d2d1335b, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:16,424 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14
2024-11-08T23:43:16,425 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 7e033ff00998e1082f711e70d2d1335b, server=a15bf3ddec7f,40483,1731109389343 in 188 msec
2024-11-08T23:43:16,428 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-08T23:43:16,428 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, ASSIGN in 351 msec
2024-11-08T23:43:16,429 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T23:43:16,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109396430"}]},"ts":"1731109396430"}
2024-11-08T23:43:16,432 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLED in hbase:meta
2024-11-08T23:43:16,433 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T23:43:16,436 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes in 1.1960 sec
2024-11-08T23:43:16,491 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,494 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109396494"}]},"ts":"1731109396494"}
2024-11-08T23:43:16,497 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLING in hbase:meta
2024-11-08T23:43:16,497 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTableMultipleTimes to state=DISABLING
2024-11-08T23:43:16,498 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes}]
2024-11-08T23:43:16,501 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, UNASSIGN}]
2024-11-08T23:43:16,503 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, UNASSIGN
2024-11-08T23:43:16,505 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=7e033ff00998e1082f711e70d2d1335b, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:16,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, UNASSIGN because future has completed
2024-11-08T23:43:16,508 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:16,508 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e033ff00998e1082f711e70d2d1335b, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:16,662 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(122): Close 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,662 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T23:43:16,662 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1722): Closing 7e033ff00998e1082f711e70d2d1335b, disabling compactions & flushes
2024-11-08T23:43:16,662 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,662 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,662 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b. after waiting 0 ms
2024-11-08T23:43:16,662 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,668 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T23:43:16,669 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.
2024-11-08T23:43:16,670 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1676): Region close journal for 7e033ff00998e1082f711e70d2d1335b:
Waiting for close lock at 1731109396662Running coprocessor pre-close hooks at 1731109396662Disabling compacts and flushes for region at 1731109396662Disabling writes for close at 1731109396662Writing region close event to WAL at 1731109396663 (+1 ms)Running coprocessor post-close hooks at 1731109396669 (+6 ms)Closed at 1731109396669
2024-11-08T23:43:16,673 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(157): Closed 7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,674 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=7e033ff00998e1082f711e70d2d1335b, regionState=CLOSED
2024-11-08T23:43:16,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7e033ff00998e1082f711e70d2d1335b, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:16,681 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18
2024-11-08T23:43:16,681 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; CloseRegionProcedure 7e033ff00998e1082f711e70d2d1335b, server=a15bf3ddec7f,40483,1731109389343 in 170 msec
2024-11-08T23:43:16,684 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17
2024-11-08T23:43:16,684 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=7e033ff00998e1082f711e70d2d1335b, UNASSIGN in 181 msec
2024-11-08T23:43:16,688 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16
2024-11-08T23:43:16,688 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes in 187 msec
2024-11-08T23:43:16,690 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109396690"}]},"ts":"1731109396690"}
2024-11-08T23:43:16,693 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLED in hbase:meta
2024-11-08T23:43:16,693 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTableMultipleTimes to state=DISABLED
2024-11-08T23:43:16,696 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes in 205 msec
2024-11-08T23:43:16,741 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(101): Disable failed with exception {}org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
2024-11-08T23:43:16,741 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(114): Disable failed with expected exception {}
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:109) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:16,742 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(126): Disable failed with expected exception {}
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:64) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:120) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:16,742 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T23:43:16,745 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTableMultipleTimes
2024-11-08T23:43:16,746 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTableMultipleTimes
2024-11-08T23:43:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] ipc.CallRunner(138): callId: 7 service: MasterService methodName: DisableTable size: 92 connection: 172.17.0.2:37578 deadline: 1731109456745, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
2024-11-08T23:43:16,747 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTableMultipleTimes failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-08T23:43:16,747 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTableMultipleTimes already disabled, so just deleting it.
2024-11-08T23:43:16,747 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTableMultipleTimes
2024-11-08T23:43:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,750 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-08T23:43:16,752 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,755 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,759 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/f1, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/f2, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/recovered.edits]
2024-11-08T23:43:16,768 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/recovered.edits/4.seqid to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/archive/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b/recovered.edits/4.seqid
2024-11-08T23:43:16,769 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testDisableTableMultipleTimes/7e033ff00998e1082f711e70d2d1335b
2024-11-08T23:43:16,769 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTableMultipleTimes regions
2024-11-08T23:43:16,773 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,776 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTableMultipleTimes from hbase:meta
2024-11-08T23:43:16,779 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTableMultipleTimes' descriptor.
2024-11-08T23:43:16,781 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,781 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTableMultipleTimes' from region states.
2024-11-08T23:43:16,781 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731109396781"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:16,784 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META
2024-11-08T23:43:16,784 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7e033ff00998e1082f711e70d2d1335b, NAME => 'testDisableTableMultipleTimes,,1731109395237.7e033ff00998e1082f711e70d2d1335b.', STARTKEY => '', ENDKEY => ''}]
2024-11-08T23:43:16,784 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTableMultipleTimes' as deleted.
2024-11-08T23:43:16,784 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731109396784"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:16,786 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTableMultipleTimes state from META
2024-11-08T23:43:16,788 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:16,789 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes in 41 msec
2024-11-08T23:43:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21
2024-11-08T23:43:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTableMultipleTimes
2024-11-08T23:43:16,856 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTableMultipleTimes completed
2024-11-08T23:43:16,865 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=224 (was 217)
Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1825474281_22 at /127.0.0.1:54754 [Waiting for operation #7]
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
	app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
	app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
	java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263)
	java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334)
	java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312)
	app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72)
	app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/data/data2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/data/data1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=119 (was 119), ProcessCount=11 (was 11), AvailableMemoryMB=3230 (was 3235)
2024-11-08T23:43:16,873 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=224, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=3230
2024-11-08T23:43:16,873 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T23:43:16,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:16,876 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T23:43:16,876 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:16,878 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T23:43:16,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741839_1015 (size=653)
2024-11-08T23:43:17,289 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf6fbf7ab1d8598703f8aae90bf92639, NAME => 'testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:17,290 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 48448b2d5448f5170de40f2655e0f6af, NAME => 'testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:17,290 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d00c625e8f6efb5b657e0fd1e1a0abd6, NAME => 'testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:17,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741840_1016 (size=67)
2024-11-08T23:43:17,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741841_1017 (size=66)
2024-11-08T23:43:17,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741842_1018 (size=67)
2024-11-08T23:43:17,304 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:17,304 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing d00c625e8f6efb5b657e0fd1e1a0abd6, disabling compactions & flushes
2024-11-08T23:43:17,305 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:17,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:17,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6. after waiting 0 ms
2024-11-08T23:43:17,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:17,305 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:17,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for d00c625e8f6efb5b657e0fd1e1a0abd6:
Waiting for close lock at 1731109397304Disabling compacts and flushes for region at 1731109397304Disabling writes for close at 1731109397305 (+1 ms)Writing region close event to WAL at 1731109397305Closed at 1731109397305
2024-11-08T23:43:17,306 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4a88ca34734a6508e6f77a4ad8ee83b0, NAME => 'testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.', STARTKEY => 'c', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1
2024-11-08T23:43:17,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741843_1019 (size=66)
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1722): Closing 48448b2d5448f5170de40f2655e0f6af, disabling compactions & flushes
2024-11-08T23:43:17,704 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af. after waiting 0 ms
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:17,704 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1676): Region close journal for 48448b2d5448f5170de40f2655e0f6af:
Waiting for close lock at 1731109397704Disabling compacts and flushes for region at 1731109397704Disabling writes for close at 1731109397704Writing region close event to WAL at 1731109397704Closed at 1731109397704
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:17,704 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing cf6fbf7ab1d8598703f8aae90bf92639, disabling compactions & flushes
2024-11-08T23:43:17,704 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:17,705 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:17,705 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639. after waiting 0 ms
2024-11-08T23:43:17,705 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:17,705 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:17,705 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf6fbf7ab1d8598703f8aae90bf92639:
Waiting for close lock at 1731109397704Disabling compacts and flushes for region at 1731109397704Disabling writes for close at 1731109397705 (+1 ms)Writing region close event to WAL at 1731109397705Closed at 1731109397705
2024-11-08T23:43:17,715 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:17,715 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 4a88ca34734a6508e6f77a4ad8ee83b0, disabling compactions & flushes
2024-11-08T23:43:17,715 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:17,715 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:17,715 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0. after waiting 0 ms
2024-11-08T23:43:17,715 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:17,715 INFO  [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:17,715 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4a88ca34734a6508e6f77a4ad8ee83b0:
Waiting for close lock at 1731109397715Disabling compacts and flushes for region at 1731109397715Disabling writes for close at 1731109397715Writing region close event to WAL at 1731109397715Closed at 1731109397715
2024-11-08T23:43:17,717 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T23:43:17,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731109397717"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109397717"}]},"ts":"1731109397717"}
2024-11-08T23:43:17,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731109397717"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109397717"}]},"ts":"1731109397717"}
2024-11-08T23:43:17,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731109397717"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109397717"}]},"ts":"1731109397717"}
2024-11-08T23:43:17,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731109397717"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731109397717"}]},"ts":"1731109397717"}
2024-11-08T23:43:17,723 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 4 regions to meta.
2024-11-08T23:43:17,725 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T23:43:17,725 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109397725"}]},"ts":"1731109397725"}
2024-11-08T23:43:17,727 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLING in hbase:meta
2024-11-08T23:43:17,728 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, ASSIGN}, {pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, ASSIGN}, {pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, ASSIGN}, {pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, ASSIGN}]
2024-11-08T23:43:17,730 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, ASSIGN
2024-11-08T23:43:17,731 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,40483,1731109389343; forceNewPlan=false, retain=false
2024-11-08T23:43:17,732 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, ASSIGN
2024-11-08T23:43:17,733 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,40483,1731109389343; forceNewPlan=false, retain=false
2024-11-08T23:43:17,734 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, ASSIGN
2024-11-08T23:43:17,735 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,40483,1731109389343; forceNewPlan=false, retain=false
2024-11-08T23:43:17,736 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, ASSIGN
2024-11-08T23:43:17,737 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, ASSIGN; state=OFFLINE, location=a15bf3ddec7f,40483,1731109389343; forceNewPlan=false, retain=false
2024-11-08T23:43:17,882 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:17,883 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:17,884 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:17,884 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPENING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:17,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, ASSIGN because future has completed
2024-11-08T23:43:17,885 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:17,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, ASSIGN because future has completed
2024-11-08T23:43:17,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, ASSIGN because future has completed
2024-11-08T23:43:17,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, ASSIGN because future has completed
2024-11-08T23:43:17,890 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:17,894 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:17,899 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:18,043 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:18,043 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a88ca34734a6508e6f77a4ad8ee83b0, NAME => 'testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.', STARTKEY => 'c', ENDKEY => ''}
2024-11-08T23:43:18,044 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,044 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:18,044 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,044 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,046 INFO  [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 4a88ca34734a6508e6f77a4ad8ee83b0 
2024-11-08T23:43:18,048 INFO  [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a88ca34734a6508e6f77a4ad8ee83b0 columnFamilyName f1
2024-11-08T23:43:18,048 DEBUG [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,048 INFO  [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] regionserver.HStore(327): Store=4a88ca34734a6508e6f77a4ad8ee83b0/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,049 INFO  [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 4a88ca34734a6508e6f77a4ad8ee83b0 
2024-11-08T23:43:18,050 INFO  [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a88ca34734a6508e6f77a4ad8ee83b0 columnFamilyName f2
2024-11-08T23:43:18,050 DEBUG [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,051 INFO  [StoreOpener-4a88ca34734a6508e6f77a4ad8ee83b0-1 {}] regionserver.HStore(327): Store=4a88ca34734a6508e6f77a4ad8ee83b0/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,051 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,052 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,053 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,054 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,054 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,054 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T23:43:18,056 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,059 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:18,059 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1114): Opened 4a88ca34734a6508e6f77a4ad8ee83b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71862457, jitterRate=0.07083405554294586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T23:43:18,060 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:18,061 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 4a88ca34734a6508e6f77a4ad8ee83b0:
Running coprocessor pre-open hook at 1731109398044Writing region info on filesystem at 1731109398044Initializing all the Stores at 1731109398045 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398045Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398046 (+1 ms)Cleaning up temporary data from old regions at 1731109398054 (+8 ms)Running coprocessor post-open hooks at 1731109398060 (+6 ms)Region opened successfully at 1731109398061 (+1 ms)
2024-11-08T23:43:18,063 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0., pid=27, masterSystemTime=1731109398038
2024-11-08T23:43:18,066 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:18,066 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:18,066 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:18,066 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => cf6fbf7ab1d8598703f8aae90bf92639, NAME => 'testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.', STARTKEY => '', ENDKEY => 'a'}
2024-11-08T23:43:18,067 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,067 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:18,067 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,067 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,067 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:18,069 INFO  [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region cf6fbf7ab1d8598703f8aae90bf92639 
2024-11-08T23:43:18,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:18,071 INFO  [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf6fbf7ab1d8598703f8aae90bf92639 columnFamilyName f1
2024-11-08T23:43:18,071 DEBUG [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,071 INFO  [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] regionserver.HStore(327): Store=cf6fbf7ab1d8598703f8aae90bf92639/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,072 INFO  [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region cf6fbf7ab1d8598703f8aae90bf92639 
2024-11-08T23:43:18,074 INFO  [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf6fbf7ab1d8598703f8aae90bf92639 columnFamilyName f2
2024-11-08T23:43:18,074 DEBUG [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,074 INFO  [StoreOpener-cf6fbf7ab1d8598703f8aae90bf92639-1 {}] regionserver.HStore(327): Store=cf6fbf7ab1d8598703f8aae90bf92639/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,075 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,075 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26
2024-11-08T23:43:18,075 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 in 186 msec
2024-11-08T23:43:18,076 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,076 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,077 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, ASSIGN in 347 msec
2024-11-08T23:43:18,078 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,078 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,078 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T23:43:18,080 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,083 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:18,083 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened cf6fbf7ab1d8598703f8aae90bf92639; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64544932, jitterRate=-0.03820556402206421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T23:43:18,084 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:18,084 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for cf6fbf7ab1d8598703f8aae90bf92639:
Running coprocessor pre-open hook at 1731109398067Writing region info on filesystem at 1731109398067Initializing all the Stores at 1731109398068 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398068Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398069 (+1 ms)Cleaning up temporary data from old regions at 1731109398078 (+9 ms)Running coprocessor post-open hooks at 1731109398084 (+6 ms)Region opened successfully at 1731109398084
2024-11-08T23:43:18,085 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639., pid=28, masterSystemTime=1731109398038
2024-11-08T23:43:18,087 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:18,087 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:18,088 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:18,088 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 48448b2d5448f5170de40f2655e0f6af, NAME => 'testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.', STARTKEY => 'b', ENDKEY => 'c'}
2024-11-08T23:43:18,088 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,088 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:18,088 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:18,089 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,089 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,090 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-08T23:43:18,091 INFO  [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 48448b2d5448f5170de40f2655e0f6af 
2024-11-08T23:43:18,092 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution'
2024-11-08T23:43:18,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:18,093 INFO  [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48448b2d5448f5170de40f2655e0f6af columnFamilyName f1
2024-11-08T23:43:18,093 DEBUG [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,093 INFO  [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] regionserver.HStore(327): Store=48448b2d5448f5170de40f2655e0f6af/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,094 INFO  [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 48448b2d5448f5170de40f2655e0f6af 
2024-11-08T23:43:18,095 INFO  [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48448b2d5448f5170de40f2655e0f6af columnFamilyName f2
2024-11-08T23:43:18,096 DEBUG [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,096 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=23
2024-11-08T23:43:18,096 INFO  [StoreOpener-48448b2d5448f5170de40f2655e0f6af-1 {}] regionserver.HStore(327): Store=48448b2d5448f5170de40f2655e0f6af/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,096 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 in 203 msec
2024-11-08T23:43:18,096 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,098 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,098 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,099 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, ASSIGN in 369 msec
2024-11-08T23:43:18,100 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,100 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,100 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T23:43:18,102 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,106 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:18,106 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 48448b2d5448f5170de40f2655e0f6af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72195535, jitterRate=0.07579730451107025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T23:43:18,106 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:18,107 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 48448b2d5448f5170de40f2655e0f6af:
Running coprocessor pre-open hook at 1731109398089Writing region info on filesystem at 1731109398089Initializing all the Stores at 1731109398091 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398091Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398091Cleaning up temporary data from old regions at 1731109398100 (+9 ms)Running coprocessor post-open hooks at 1731109398106 (+6 ms)Region opened successfully at 1731109398106
2024-11-08T23:43:18,107 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af., pid=30, masterSystemTime=1731109398038
2024-11-08T23:43:18,110 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:18,110 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:18,110 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:18,111 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => d00c625e8f6efb5b657e0fd1e1a0abd6, NAME => 'testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.', STARTKEY => 'a', ENDKEY => 'b'}
2024-11-08T23:43:18,111 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:18,111 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,112 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T23:43:18,112 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,112 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,114 INFO  [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region d00c625e8f6efb5b657e0fd1e1a0abd6 
2024-11-08T23:43:18,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:18,116 INFO  [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d00c625e8f6efb5b657e0fd1e1a0abd6 columnFamilyName f1
2024-11-08T23:43:18,117 DEBUG [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,117 INFO  [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] regionserver.HStore(327): Store=d00c625e8f6efb5b657e0fd1e1a0abd6/f1,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,117 INFO  [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region d00c625e8f6efb5b657e0fd1e1a0abd6 
2024-11-08T23:43:18,119 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=25
2024-11-08T23:43:18,119 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 in 218 msec
2024-11-08T23:43:18,120 INFO  [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d00c625e8f6efb5b657e0fd1e1a0abd6 columnFamilyName f2
2024-11-08T23:43:18,120 DEBUG [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T23:43:18,121 INFO  [StoreOpener-d00c625e8f6efb5b657e0fd1e1a0abd6-1 {}] regionserver.HStore(327): Store=d00c625e8f6efb5b657e0fd1e1a0abd6/f2,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T23:43:18,121 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, ASSIGN in 391 msec
2024-11-08T23:43:18,121 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,122 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,123 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,124 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,124 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,125 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead.
2024-11-08T23:43:18,127 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,130 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T23:43:18,131 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened d00c625e8f6efb5b657e0fd1e1a0abd6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62109654, jitterRate=-0.07449403405189514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864}
2024-11-08T23:43:18,131 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:18,131 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for d00c625e8f6efb5b657e0fd1e1a0abd6:
Running coprocessor pre-open hook at 1731109398112Writing region info on filesystem at 1731109398112Initializing all the Stores at 1731109398114 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398114Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731109398114Cleaning up temporary data from old regions at 1731109398124 (+10 ms)Running coprocessor post-open hooks at 1731109398131 (+7 ms)Region opened successfully at 1731109398131
2024-11-08T23:43:18,132 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6., pid=29, masterSystemTime=1731109398038
2024-11-08T23:43:18,135 DEBUG [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:18,135 INFO  [RS_OPEN_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:18,136 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPEN, openSeqNum=2, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:18,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:18,143 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=24
2024-11-08T23:43:18,143 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 in 246 msec
2024-11-08T23:43:18,146 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22
2024-11-08T23:43:18,146 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, ASSIGN in 415 msec
2024-11-08T23:43:18,147 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T23:43:18,147 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109398147"}]},"ts":"1731109398147"}
2024-11-08T23:43:18,150 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLED in hbase:meta
2024-11-08T23:43:18,151 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T23:43:18,153 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution in 1.2780 sec
2024-11-08T23:43:18,375 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true
2024-11-08T23:43:18,377 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,379 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:18,379 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,379 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:18,380 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:18,628 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,629 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:18,629 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:18,629 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:18,629 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:18,629 ERROR [Time-limited test {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[HFileArchiver-1,5,PEWorkerGroup]
    Thread[HFileArchiver-2,5,PEWorkerGroup]
2024-11-08T23:43:18,630 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:18,630 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:18,630 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:18,631 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:18,632 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:18,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[]
2024-11-08T23:43:18,637 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:18,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:18,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:18,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:18,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:18,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:18,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:18,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,641 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:18,641 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec
2024-11-08T23:43:18,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,644 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:18,645 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:18,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:18,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:18,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:18,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:18,649 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109428649
2024-11-08T23:43:18,649 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:18,649 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,656 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:18,656 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109458656
2024-11-08T23:43:18,656 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109518656
2024-11-08T23:43:18,656 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec
2024-11-08T23:43:18,656 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:18,658 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:18,658 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,658 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:18,658 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:18,720 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-08T23:43:18,906 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:18,907 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:18,908 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:18,909 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:18,910 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:18,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0]
2024-11-08T23:43:18,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:18,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:18,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:18,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:18,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:18,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:18,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:18,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:18,916 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-08T23:43:18,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,917 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:18,917 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:18,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:18,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:18,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:18,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:18,920 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109428920
2024-11-08T23:43:18,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:18,920 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:18,926 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:18,926 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109458926
2024-11-08T23:43:18,927 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109518927
2024-11-08T23:43:18,927 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec
2024-11-08T23:43:18,927 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:18,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109398928"}]},"ts":"1731109398928"}
2024-11-08T23:43:18,932 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta
2024-11-08T23:43:18,932 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING
2024-11-08T23:43:18,932 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:18,932 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:18,932 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:18,932 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:19,076 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-08T23:43:19,077 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-08T23:43:19,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-08T23:43:19,078 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-08T23:43:19,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-08T23:43:19,079 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-08T23:43:19,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution
2024-11-08T23:43:19,080 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution Metrics about Tables on a single HBase RegionServer
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:19,177 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:19,178 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:19,179 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:19,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1]
2024-11-08T23:43:19,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:19,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:19,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:19,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:19,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,184 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:19,184 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:19,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,185 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:19,185 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:19,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,188 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109429188
2024-11-08T23:43:19,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:19,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,194 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:19,194 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109459194
2024-11-08T23:43:19,194 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109519194
2024-11-08T23:43:19,194 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T23:43:19,194 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109399194"}]},"ts":"1731109399194"}
2024-11-08T23:43:19,196 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta
2024-11-08T23:43:19,197 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING
2024-11-08T23:43:19,197 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:19,198 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}]
2024-11-08T23:43:19,198 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:19,198 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,198 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,198 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:19,444 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:19,445 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:19,446 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:19,448 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2]
2024-11-08T23:43:19,448 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:19,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:19,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:19,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:19,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,451 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:19,451 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:19,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,452 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:19,452 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:19,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,456 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109429456
2024-11-08T23:43:19,457 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,457 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:19,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,463 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:19,463 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109459463
2024-11-08T23:43:19,463 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109519463
2024-11-08T23:43:19,463 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec
2024-11-08T23:43:19,463 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}]
2024-11-08T23:43:19,463 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:19,467 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN}]
2024-11-08T23:43:19,468 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:19,468 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,468 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,468 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:19,713 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,713 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:19,713 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,714 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:19,714 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:19,714 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:19,714 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:19,714 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:19,714 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:19,715 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:19,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:19,718 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[]
2024-11-08T23:43:19,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:19,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:19,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:19,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:19,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,721 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:19,721 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:19,721 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec
2024-11-08T23:43:19,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,722 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:19,723 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:19,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,726 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109429726
2024-11-08T23:43:19,726 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:19,726 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,731 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:19,731 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109459731
2024-11-08T23:43:19,731 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109519731
2024-11-08T23:43:19,731 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T23:43:19,731 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN}]
2024-11-08T23:43:19,731 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:19,734 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:19,736 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:19,737 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:19,738 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:19,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:19,738 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:19,740 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:19,741 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:19,741 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:19,741 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:19,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:19,741 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:19,741 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,741 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:19,741 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f18acbd88f8@71a654fb rejected from java.util.concurrent.ThreadPoolExecutor@364fc46f[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:19,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN because future has completed
2024-11-08T23:43:19,743 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f18acbd88f8@2f4cd39b rejected from java.util.concurrent.ThreadPoolExecutor@364fc46f[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:19,836 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:19,836 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:19,939 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:19,939 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:19,981 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:19,982 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:19,983 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:19,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:19,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:19,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:19,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:19,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:19,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:19,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:19,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:19,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:19,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:19,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:19,989 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:19,990 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:19,990 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:19,991 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:19,991 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:19,991 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:19,991 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:19,991 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec
2024-11-08T23:43:19,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:19,992 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:19,992 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:19,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:19,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:19,995 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639 to restore RIT
2024-11-08T23:43:19,995 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6 to restore RIT
2024-11-08T23:43:19,995 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:19,995 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:19,995 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109429995
2024-11-08T23:43:19,996 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:19,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:19,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:19,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:19,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,001 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:20,001 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109460001
2024-11-08T23:43:20,001 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109520001
2024-11-08T23:43:20,001 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T23:43:20,001 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,003 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,003 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,004 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,004 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN because future has completed
2024-11-08T23:43:20,006 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:20,006 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:20,006 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:20,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN because future has completed
2024-11-08T23:43:20,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:20,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:20,010 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:20,010 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:20,010 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:20,010 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,010 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,010 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:20,042 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,042 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,144 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,144 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,159 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(122): Close cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:20,159 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T23:43:20,160 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1722): Closing cf6fbf7ab1d8598703f8aae90bf92639, disabling compactions & flushes
2024-11-08T23:43:20,160 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:20,160 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:20,160 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639. after waiting 0 ms
2024-11-08T23:43:20,160 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:20,166 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T23:43:20,167 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.
2024-11-08T23:43:20,167 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1676): Region close journal for cf6fbf7ab1d8598703f8aae90bf92639:
Waiting for close lock at 1731109400160Running coprocessor pre-close hooks at 1731109400160Disabling compacts and flushes for region at 1731109400160Disabling writes for close at 1731109400160Writing region close event to WAL at 1731109400160Running coprocessor post-close hooks at 1731109400166 (+6 ms)Closed at 1731109400167 (+1 ms)
2024-11-08T23:43:20,169 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(157): Closed cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:20,247 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,247 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,251 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,251 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:20,251 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,251 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:20,251 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:20,252 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:20,252 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:20,252 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:20,252 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:20,253 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,255 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:20,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:20,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:20,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:20,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,259 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,259 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,259 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,259 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,259 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:20,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:20,260 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:20,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:20,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:20,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:20,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:20,263 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639 to restore RIT
2024-11-08T23:43:20,263 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6 to restore RIT
2024-11-08T23:43:20,263 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:20,263 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:20,263 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109430263
2024-11-08T23:43:20,263 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:20,263 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,264 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,264 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,264 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,269 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:20,269 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109460269
2024-11-08T23:43:20,269 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109520269
2024-11-08T23:43:20,269 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T23:43:20,271 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED
2024-11-08T23:43:20,272 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:20,273 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,275 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:20,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN because future has completed
2024-11-08T23:43:20,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:20,277 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=33
2024-11-08T23:43:20,277 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 in 269 msec
2024-11-08T23:43:20,277 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:20,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,277 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,278 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:20,350 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,350 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,452 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,452 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,519 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,519 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:20,519 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,519 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:20,519 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:20,520 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:20,520 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:20,520 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:20,520 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:20,521 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:20,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:20,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:20,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:20,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,525 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,527 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,527 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,527 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,527 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,527 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,527 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,527 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:20,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,528 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:20,528 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:20,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:20,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:20,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:20,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:20,532 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639 to restore RIT
2024-11-08T23:43:20,532 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6 to restore RIT
2024-11-08T23:43:20,532 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:20,532 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:20,532 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109430532
2024-11-08T23:43:20,532 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:20,532 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,533 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,533 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,533 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,537 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:20,537 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109460537
2024-11-08T23:43:20,537 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109520537
2024-11-08T23:43:20,537 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T23:43:20,537 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,539 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:20,541 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN in 808 msec
2024-11-08T23:43:20,541 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,542 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,542 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN because future has completed
2024-11-08T23:43:20,544 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:20,544 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:20,544 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:20,544 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,544 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,544 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:20,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:20,545 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f18acbd88f8@71cad296 rejected from java.util.concurrent.ThreadPoolExecutor@4010af05[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:20,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:20,546 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f18acbd88f8@60becc21 rejected from java.util.concurrent.ThreadPoolExecutor@4010af05[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:20,555 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,555 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,658 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,658 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,761 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,761 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,787 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,787 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:20,787 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,787 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:20,787 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:20,788 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:20,788 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:20,788 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:20,788 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:20,789 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:20,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:20,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:20,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:20,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:20,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:20,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,794 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,794 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:20,794 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:20,795 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:20,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,795 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:20,796 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:20,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:20,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:20,798 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6 to restore RIT
2024-11-08T23:43:20,798 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:20,798 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:20,799 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109430799
2024-11-08T23:43:20,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:20,799 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:20,799 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:20,799 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:20,804 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:20,804 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109460804
2024-11-08T23:43:20,804 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109520804
2024-11-08T23:43:20,804 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec
2024-11-08T23:43:20,804 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,805 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,805 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:20,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN because future has completed
2024-11-08T23:43:20,807 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:20,807 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:20,808 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:20,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:20,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:20,812 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:20,812 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:20,812 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:20,812 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:20,812 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:20,812 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:20,864 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,961 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:20,961 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T23:43:20,961 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing d00c625e8f6efb5b657e0fd1e1a0abd6, disabling compactions & flushes
2024-11-08T23:43:20,961 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:20,961 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:20,962 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6. after waiting 0 ms
2024-11-08T23:43:20,962 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:20,966 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:20,967 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T23:43:20,968 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.
2024-11-08T23:43:20,969 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for d00c625e8f6efb5b657e0fd1e1a0abd6:
Waiting for close lock at 1731109400961Running coprocessor pre-close hooks at 1731109400961Disabling compacts and flushes for region at 1731109400961Disabling writes for close at 1731109400962 (+1 ms)Writing region close event to WAL at 1731109400962Running coprocessor post-close hooks at 1731109400968 (+6 ms)Closed at 1731109400968
2024-11-08T23:43:20,972 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:21,054 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 9 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,054 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:21,054 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,054 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:21,054 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:21,054 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:21,055 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:21,055 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:21,055 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:21,056 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:21,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:21,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:21,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:21,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,061 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,061 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,061 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,062 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:21,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,062 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:21,063 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:21,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,065 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6 to restore RIT
2024-11-08T23:43:21,065 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:21,066 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:21,066 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109431066
2024-11-08T23:43:21,066 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:21,066 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,066 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,066 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,073 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:21,073 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109461073
2024-11-08T23:43:21,073 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109521073
2024-11-08T23:43:21,073 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec
2024-11-08T23:43:21,075 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED
2024-11-08T23:43:21,076 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,077 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:21,078 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:21,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:21,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:21,081 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=34
2024-11-08T23:43:21,081 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 in 271 msec
2024-11-08T23:43:21,081 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:21,081 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,081 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,081 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:21,175 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,175 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,277 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,277 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,323 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 10 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,323 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:21,323 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,323 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:21,324 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:21,324 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:21,324 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:21,324 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:21,324 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:21,325 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:21,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:21,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:21,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:21,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,330 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,330 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,331 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:21,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,331 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:21,332 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:21,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,334 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6 to restore RIT
2024-11-08T23:43:21,334 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:21,334 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:21,336 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109431336
2024-11-08T23:43:21,336 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,336 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:21,336 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,336 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,341 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:21,341 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109461341
2024-11-08T23:43:21,341 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109521341
2024-11-08T23:43:21,341 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T23:43:21,341 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,342 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:21,344 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN in 1.6110 sec
2024-11-08T23:43:21,344 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,345 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:21,347 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:21,347 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:21,347 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:21,347 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,347 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:21,348 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:21,348 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f18acbd88f8@443981bf rejected from java.util.concurrent.ThreadPoolExecutor@7e775ec6[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]
	at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
	at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
	at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T23:43:21,380 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,483 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,585 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,591 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 11 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,591 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:21,591 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,591 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:21,591 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:21,592 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:21,592 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:21,592 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:21,592 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:21,594 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec
2024-11-08T23:43:21,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:21,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:21,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:21,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:21,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,602 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,602 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,602 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec
2024-11-08T23:43:21,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:21,604 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:21,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,608 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:21,608 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:21,608 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109431608
2024-11-08T23:43:21,608 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:21,608 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,608 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,613 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:21,613 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109461613
2024-11-08T23:43:21,613 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109521613
2024-11-08T23:43:21,613 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T23:43:21,613 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,614 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN because future has completed
2024-11-08T23:43:21,617 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:21,617 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:21,617 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:21,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:21,622 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:21,622 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:21,622 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:21,622 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,622 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,623 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:21,771 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(122): Close 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:21,772 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T23:43:21,772 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1722): Closing 48448b2d5448f5170de40f2655e0f6af, disabling compactions & flushes
2024-11-08T23:43:21,772 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:21,772 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:21,772 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af. after waiting 0 ms
2024-11-08T23:43:21,772 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:21,778 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T23:43:21,779 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.
2024-11-08T23:43:21,779 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1676): Region close journal for 48448b2d5448f5170de40f2655e0f6af:
Waiting for close lock at 1731109401772Running coprocessor pre-close hooks at 1731109401772Disabling compacts and flushes for region at 1731109401772Disabling writes for close at 1731109401772Writing region close event to WAL at 1731109401772Running coprocessor post-close hooks at 1731109401779 (+7 ms)Closed at 1731109401779
2024-11-08T23:43:21,781 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(157): Closed 48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:21,863 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 12 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,863 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:21,863 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,863 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:21,863 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:21,864 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:21,864 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:21,864 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:21,864 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:21,865 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:21,866 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:21,867 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:21,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:21,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:21,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:21,869 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,869 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,870 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:21,870 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:21,870 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:21,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,871 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:21,871 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:21,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:21,873 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,873 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:21,874 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:21,874 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:21,874 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109431874
2024-11-08T23:43:21,874 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:21,874 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,875 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:21,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:21,879 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:21,879 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109461879
2024-11-08T23:43:21,879 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109521879
2024-11-08T23:43:21,879 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec
2024-11-08T23:43:21,880 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED
2024-11-08T23:43:21,882 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:21,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:21,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:21,884 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:21,886 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=35
2024-11-08T23:43:21,886 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 in 267 msec
2024-11-08T23:43:21,886 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:21,887 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:21,887 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:21,887 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:21,891 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:21,994 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:22,096 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a15bf3ddec7f,40483,1731109389343, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-08T23:43:22,129 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 13 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,129 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:22,129 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,129 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:22,129 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:22,130 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:22,130 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:22,130 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:22,130 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:22,131 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:22,132 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:22,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:22,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:22,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:22,133 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:22,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,135 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,136 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:22,136 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,136 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,136 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T23:43:22,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,137 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:22,137 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:22,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,139 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af to restore RIT
2024-11-08T23:43:22,139 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:22,140 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109432140
2024-11-08T23:43:22,140 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:22,140 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:22,140 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,143 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:22,143 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109462143
2024-11-08T23:43:22,143 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109522143
2024-11-08T23:43:22,143 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:22,143 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:22,145 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:22,146 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN in 2.4140 sec
2024-11-08T23:43:22,147 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:22,149 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:22,149 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:22,149 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:22,149 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,149 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,150 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 14 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:22,394 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:22,395 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[]
2024-11-08T23:43:22,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:22,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:22,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:22,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:22,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:22,401 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,401 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:22,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,402 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:22,402 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:22,402 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:22,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,405 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:22,405 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109432405
2024-11-08T23:43:22,405 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:22,405 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,409 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:22,409 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109462409
2024-11-08T23:43:22,409 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109522409
2024-11-08T23:43:22,409 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:22,409 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSING, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN because future has completed
2024-11-08T23:43:22,412 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false
2024-11-08T23:43:22,413 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343}]
2024-11-08T23:43:22,413 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:22,566 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:22,566 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false
2024-11-08T23:43:22,566 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 4a88ca34734a6508e6f77a4ad8ee83b0, disabling compactions & flushes
2024-11-08T23:43:22,566 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:22,566 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:22,566 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0. after waiting 0 ms
2024-11-08T23:43:22,566 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:22,572 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1
2024-11-08T23:43:22,573 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.
2024-11-08T23:43:22,573 DEBUG [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 4a88ca34734a6508e6f77a4ad8ee83b0:
Waiting for close lock at 1731109402566Running coprocessor pre-close hooks at 1731109402566Disabling compacts and flushes for region at 1731109402566Disabling writes for close at 1731109402566Writing region close event to WAL at 1731109402567 (+1 ms)Running coprocessor post-close hooks at 1731109402573 (+6 ms)Closed at 1731109402573
2024-11-08T23:43:22,576 INFO  [RS_CLOSE_REGION-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:22,576 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED
2024-11-08T23:43:22,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:22,580 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:22,580 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,580 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,580 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:22,659 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 15 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,659 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:22,660 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:22,661 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:22,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18]
2024-11-08T23:43:22,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:22,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:22,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:22,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:22,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,665 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:22,666 INFO  [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,667 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,667 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:22,667 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:22,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,667 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:22,667 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:22,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,671 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:22,672 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109432672
2024-11-08T23:43:22,672 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:22,672 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,676 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:22,676 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109462676
2024-11-08T23:43:22,676 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109522676
2024-11-08T23:43:22,676 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:22,676 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, regionLocation=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 because future has completed
2024-11-08T23:43:22,680 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:22,683 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=36
2024-11-08T23:43:22,683 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 in 268 msec
2024-11-08T23:43:22,683 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:22,683 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,683 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,683 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:22,926 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 16 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,926 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:22,926 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,926 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:22,926 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:22,927 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:22,927 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:22,927 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:22,927 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:22,928 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T23:43:22,929 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:22,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:22,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:22,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:22,930 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:22,931 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:22,932 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T23:43:22,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,933 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:22,933 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:22,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:22,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:22,936 INFO  [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0 to restore RIT
2024-11-08T23:43:22,937 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109432936
2024-11-08T23:43:22,937 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:22,937 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:22,940 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:22,940 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109462940
2024-11-08T23:43:22,940 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109522940
2024-11-08T23:43:22,940 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:22,941 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:22,942 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:22,944 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=32
2024-11-08T23:43:22,944 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN in 3.2110 sec
2024-11-08T23:43:22,945 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:22,945 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:22,945 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:22,945 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 17 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:23,191 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:23,192 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:23,193 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T23:43:23,194 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:23,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:23,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:23,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:23,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,197 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T23:43:23,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,198 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:23,198 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:23,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,200 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,200 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,201 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109433201
2024-11-08T23:43:23,201 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:23,201 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,205 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:23,205 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109463205
2024-11-08T23:43:23,205 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109523205
2024-11-08T23:43:23,205 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:23,205 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:23,208 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31
2024-11-08T23:43:23,208 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution in 3.7420 sec
2024-11-08T23:43:23,209 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:23,209 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,209 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,209 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 18 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:23,455 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:23,456 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:23,456 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:23,457 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T23:43:23,458 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T23:43:23,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:23,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:23,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:23,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:23,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,462 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,463 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:23,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,463 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:23,464 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:23,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,466 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,466 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,467 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109433467
2024-11-08T23:43:23,467 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:23,467 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,470 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:23,470 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109463470
2024-11-08T23:43:23,470 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109523470
2024-11-08T23:43:23,470 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:23,470 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:23,472 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109403472"}]},"ts":"1731109403472"}
2024-11-08T23:43:23,474 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta
2024-11-08T23:43:23,474 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED
2024-11-08T23:43:23,474 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:23,474 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,474 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,474 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:23,613 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 19 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:23,721 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:23,722 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T23:43:23,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:23,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:23,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:23,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:23,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:23,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:23,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,728 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-08T23:43:23,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,729 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:23,730 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:23,731 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:23,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,733 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109433733
2024-11-08T23:43:23,733 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:23,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,737 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:23,737 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109463737
2024-11-08T23:43:23,737 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109523737
2024-11-08T23:43:23,737 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:23,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731109403737"}]},"ts":"1731109403737"}
2024-11-08T23:43:23,741 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta
2024-11-08T23:43:23,741 INFO  [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED
2024-11-08T23:43:23,741 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:23,742 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false
2024-11-08T23:43:23,742 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,742 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,742 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null
java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:23,987 INFO  [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 20 exec state=pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,987 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10
2024-11-08T23:43:23,988 INFO  [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T23:43:23,989 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-08T23:43:23,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22, 23]
2024-11-08T23:43:23,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21]
2024-11-08T23:43:23,990 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19]
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable
2024-11-08T23:43:23,991 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable
2024-11-08T23:43:23,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes
2024-11-08T23:43:23,992 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=cf6fbf7ab1d8598703f8aae90bf92639, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=d00c625e8f6efb5b657e0fd1e1a0abd6, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=48448b2d5448f5170de40f2655e0f6af, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=4a88ca34734a6508e6f77a4ad8ee83b0, UNASSIGN didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure cf6fbf7ab1d8598703f8aae90bf92639, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure d00c625e8f6efb5b657e0fd1e1a0abd6, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,993 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 48448b2d5448f5170de40f2655e0f6af, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,994 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a88ca34734a6508e6f77a4ad8ee83b0, server=a15bf3ddec7f,40483,1731109389343 didn't hold the lock before restarting, skip acquiring lock.
2024-11-08T23:43:23,994 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T23:43:23,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:23,994 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a15bf3ddec7f,40483,1731109389343, table=hbase:meta, region=1588230740
2024-11-08T23:43:23,994 INFO  [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a15bf3ddec7f,40483,1731109389343, state=OPEN
2024-11-08T23:43:23,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T23:43:23,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T23:43:23,997 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731109433997
2024-11-08T23:43:23,997 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-08T23:43:23,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf6fbf7ab1d8598703f8aae90bf92639, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:24,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d00c625e8f6efb5b657e0fd1e1a0abd6, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:24,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48448b2d5448f5170de40f2655e0f6af, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:24,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a88ca34734a6508e6f77a4ad8ee83b0, regionState=CLOSED, lastHost=a15bf3ddec7f,40483,1731109389343, regionLocation=a15bf3ddec7f,40483,1731109389343, openSeqNum=2
2024-11-08T23:43:24,001 INFO  [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-08T23:43:24,001 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731109464001
2024-11-08T23:43:24,001 INFO  [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731109524001
2024-11-08T23:43:24,001 INFO  [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-08T23:43:24,001 WARN  [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true
2024-11-08T23:43:24,004 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution in 5.6250 sec
2024-11-08T23:43:24,251 WARN  [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false
2024-11-08T23:43:24,253 INFO  [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,254 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testRecoveryAndDoubleExecution
2024-11-08T23:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?]
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-08T23:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] ipc.CallRunner(138): callId: 12 service: MasterService methodName: DisableTable size: 93 connection: 172.17.0.2:37578 deadline: 1731109464254, exception=org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
2024-11-08T23:43:24,255 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testRecoveryAndDoubleExecution failed with org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution
	at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163)
	at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.<init>(DisableTableProcedure.java:78)
	at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831)
	at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137)
	at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815)
	at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-08T23:43:24,255 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testRecoveryAndDoubleExecution already disabled, so just deleting it.
2024-11-08T23:43:24,256 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testRecoveryAndDoubleExecution
2024-11-08T23:43:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,258 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-08T23:43:24,259 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:24,267 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:24,267 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:24,267 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:24,271 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/f1, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/f2, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/recovered.edits]
2024-11-08T23:43:24,271 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/f1, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/f2, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/recovered.edits]
2024-11-08T23:43:24,271 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/f1, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/f2, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/recovered.edits]
2024-11-08T23:43:24,271 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/f1, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/f2, FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/recovered.edits]
2024-11-08T23:43:24,286 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/recovered.edits/4.seqid to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/archive/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6/recovered.edits/4.seqid
2024-11-08T23:43:24,286 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/recovered.edits/4.seqid to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/archive/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639/recovered.edits/4.seqid
2024-11-08T23:43:24,287 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/recovered.edits/4.seqid to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/archive/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af/recovered.edits/4.seqid
2024-11-08T23:43:24,288 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/cf6fbf7ab1d8598703f8aae90bf92639
2024-11-08T23:43:24,288 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/d00c625e8f6efb5b657e0fd1e1a0abd6
2024-11-08T23:43:24,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/48448b2d5448f5170de40f2655e0f6af
2024-11-08T23:43:24,289 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/recovered.edits/4.seqid to hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/archive/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0/recovered.edits/4.seqid
2024-11-08T23:43:24,290 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/default/testRecoveryAndDoubleExecution/4a88ca34734a6508e6f77a4ad8ee83b0
2024-11-08T23:43:24,290 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testRecoveryAndDoubleExecution regions
2024-11-08T23:43:24,293 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,297 WARN  [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 4 rows of testRecoveryAndDoubleExecution from hbase:meta
2024-11-08T23:43:24,300 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testRecoveryAndDoubleExecution' descriptor.
2024-11-08T23:43:24,302 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,302 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testRecoveryAndDoubleExecution' from region states.
2024-11-08T23:43:24,302 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731109404302"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:24,302 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731109404302"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:24,302 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731109404302"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:24,302 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731109404302"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:24,306 INFO  [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 4 regions from META
2024-11-08T23:43:24,306 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cf6fbf7ab1d8598703f8aae90bf92639, NAME => 'testRecoveryAndDoubleExecution,,1731109396873.cf6fbf7ab1d8598703f8aae90bf92639.', STARTKEY => '', ENDKEY => 'a'}, {ENCODED => d00c625e8f6efb5b657e0fd1e1a0abd6, NAME => 'testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6.', STARTKEY => 'a', ENDKEY => 'b'}, {ENCODED => 48448b2d5448f5170de40f2655e0f6af, NAME => 'testRecoveryAndDoubleExecution,b,1731109396873.48448b2d5448f5170de40f2655e0f6af.', STARTKEY => 'b', ENDKEY => 'c'}, {ENCODED => 4a88ca34734a6508e6f77a4ad8ee83b0, NAME => 'testRecoveryAndDoubleExecution,c,1731109396873.4a88ca34734a6508e6f77a4ad8ee83b0.', STARTKEY => 'c', ENDKEY => ''}]
2024-11-08T23:43:24,306 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testRecoveryAndDoubleExecution' as deleted.
2024-11-08T23:43:24,306 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731109404306"}]},"ts":"9223372036854775807"}
2024-11-08T23:43:24,308 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testRecoveryAndDoubleExecution state from META
2024-11-08T23:43:24,309 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution
2024-11-08T23:43:24,311 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution in 53 msec
2024-11-08T23:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42
2024-11-08T23:43:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testRecoveryAndDoubleExecution
2024-11-08T23:43:24,367 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testRecoveryAndDoubleExecution completed
2024-11-08T23:43:24,376 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=219 (was 224), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=100 (was 119), ProcessCount=11 (was 11), AvailableMemoryMB=3207 (was 3230)
2024-11-08T23:43:24,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-08T23:43:24,377 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-08T23:43:24,377 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T23:43:24,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T23:43:24,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T23:43:24,383 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-08T23:43:24,383 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-08T23:43:24,383 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=68020250, stopped=false
2024-11-08T23:43:24,383 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a15bf3ddec7f,46285,1731109388673
2024-11-08T23:43:24,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T23:43:24,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T23:43:24,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:24,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:24,386 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:24,386 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-08T23:43:24,386 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T23:43:24,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T23:43:24,386 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a15bf3ddec7f,40483,1731109389343' *****
2024-11-08T23:43:24,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T23:43:24,386 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T23:43:24,387 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T23:43:24,387 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-08T23:43:24,387 INFO  [RS:0;a15bf3ddec7f:40483 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-08T23:43:24,387 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-08T23:43:24,387 INFO  [RS:0;a15bf3ddec7f:40483 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-08T23:43:24,387 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(959): stopping server a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:24,387 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T23:43:24,387 INFO  [RS:0;a15bf3ddec7f:40483 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a15bf3ddec7f:40483.
2024-11-08T23:43:24,388 DEBUG [RS:0;a15bf3ddec7f:40483 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T23:43:24,388 DEBUG [RS:0;a15bf3ddec7f:40483 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T23:43:24,388 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-08T23:43:24,388 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-08T23:43:24,388 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-08T23:43:24,388 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-08T23:43:24,389 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-08T23:43:24,389 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-08T23:43:24,389 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-08T23:43:24,389 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-08T23:43:24,389 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:24,389 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-08T23:43:24,389 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-08T23:43:24,389 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-08T23:43:24,392 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=25.04 KB heapSize=40.99 KB
2024-11-08T23:43:24,437 INFO  [regionserver/a15bf3ddec7f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T23:43:24,458 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/info/e276d085b1fc41c6b2c9939aaba03405 is 108, key is testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6./info:/1731109404302/DeleteFamily/seqid=0
2024-11-08T23:43:24,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741844_1020 (size=6669)
2024-11-08T23:43:24,589 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:24,790 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:24,870 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.21 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/info/e276d085b1fc41c6b2c9939aaba03405
2024-11-08T23:43:24,930 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/ns/9122daa90adf4ad1b03656d25a1d73fe is 106, key is testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6./ns:/1731109404293/DeleteFamily/seqid=0
2024-11-08T23:43:24,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741845_1021 (size=6009)
2024-11-08T23:43:24,990 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:25,131 INFO  [regionserver/a15bf3ddec7f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-08T23:43:25,131 INFO  [regionserver/a15bf3ddec7f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-08T23:43:25,190 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:25,336 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/ns/9122daa90adf4ad1b03656d25a1d73fe
2024-11-08T23:43:25,361 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/rep_barrier/a8b71b6ca2c24a9894885d4dc3ddaf1c is 115, key is testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6./rep_barrier:/1731109404293/DeleteFamily/seqid=0
2024-11-08T23:43:25,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741846_1022 (size=6121)
2024-11-08T23:43:25,390 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:25,591 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:25,767 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=648 B at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/rep_barrier/a8b71b6ca2c24a9894885d4dc3ddaf1c
2024-11-08T23:43:25,791 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:25,801 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/table/2e3895b704514c97a09861d46a9476da is 109, key is testRecoveryAndDoubleExecution,a,1731109396873.d00c625e8f6efb5b657e0fd1e1a0abd6./table:/1731109404293/DeleteFamily/seqid=0
2024-11-08T23:43:25,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741847_1023 (size=6158)
2024-11-08T23:43:25,991 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:26,191 DEBUG [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T23:43:26,208 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.55 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/table/2e3895b704514c97a09861d46a9476da
2024-11-08T23:43:26,218 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/info/e276d085b1fc41c6b2c9939aaba03405 as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/info/e276d085b1fc41c6b2c9939aaba03405
2024-11-08T23:43:26,226 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/info/e276d085b1fc41c6b2c9939aaba03405, entries=12, sequenceid=77, filesize=6.5 K
2024-11-08T23:43:26,228 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/ns/9122daa90adf4ad1b03656d25a1d73fe as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/ns/9122daa90adf4ad1b03656d25a1d73fe
2024-11-08T23:43:26,235 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/ns/9122daa90adf4ad1b03656d25a1d73fe, entries=8, sequenceid=77, filesize=5.9 K
2024-11-08T23:43:26,236 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/rep_barrier/a8b71b6ca2c24a9894885d4dc3ddaf1c as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/rep_barrier/a8b71b6ca2c24a9894885d4dc3ddaf1c
2024-11-08T23:43:26,244 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/rep_barrier/a8b71b6ca2c24a9894885d4dc3ddaf1c, entries=6, sequenceid=77, filesize=6.0 K
2024-11-08T23:43:26,245 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/.tmp/table/2e3895b704514c97a09861d46a9476da as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/table/2e3895b704514c97a09861d46a9476da
2024-11-08T23:43:26,253 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/table/2e3895b704514c97a09861d46a9476da, entries=9, sequenceid=77, filesize=6.0 K
2024-11-08T23:43:26,258 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~25.04 KB/25645, heapSize ~40.93 KB/41912, currentSize=0 B/0 for 1588230740 in 1865ms, sequenceid=77, compaction requested=false
2024-11-08T23:43:26,266 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/data/hbase/meta/1588230740/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1
2024-11-08T23:43:26,267 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-08T23:43:26,267 INFO  [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-08T23:43:26,267 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731109404389Running coprocessor pre-close hooks at 1731109404389Disabling compacts and flushes for region at 1731109404389Disabling writes for close at 1731109404389Obtaining lock to block concurrent updates at 1731109404392 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1731109404392Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=25645, getHeapSize=41912, getOffHeapSize=0, getCellsCount=202 at 1731109404400 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731109404401 (+1 ms)Flushing 1588230740/info: creating writer at 1731109404404 (+3 ms)Flushing 1588230740/info: appending metadata at 1731109404448 (+44 ms)Flushing 1588230740/info: closing flushed file at 1731109404450 (+2 ms)Flushing 1588230740/ns: creating writer at 1731109404913 (+463 ms)Flushing 1588230740/ns: appending metadata at 1731109404929 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731109404929Flushing 1588230740/rep_barrier: creating writer at 1731109405344 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731109405360 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731109405360Flushing 1588230740/table: creating writer at 1731109405776 (+416 ms)Flushing 1588230740/table: appending metadata at 1731109405800 (+24 ms)Flushing 1588230740/table: closing flushed file at 1731109405800Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13b633a3: reopening flushed file at 1731109406216 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d87bc4c: reopening flushed file at 1731109406227 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74962b31: reopening flushed file at 1731109406235 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3610fba6: reopening flushed file at 1731109406244 (+9 ms)Finished flush of dataSize ~25.04 KB/25645, heapSize ~40.93 KB/41912, currentSize=0 B/0 for 1588230740 in 1865ms, sequenceid=77, compaction requested=false at 1731109406258 (+14 ms)Writing region close event to WAL at 1731109406261 (+3 ms)Running coprocessor post-close hooks at 1731109406267 (+6 ms)Closed at 1731109406267
2024-11-08T23:43:26,267 DEBUG [RS_CLOSE_META-regionserver/a15bf3ddec7f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-08T23:43:26,392 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(976): stopping server a15bf3ddec7f,40483,1731109389343; all regions closed.
2024-11-08T23:43:26,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741834_1010 (size=30181)
2024-11-08T23:43:26,400 DEBUG [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/oldWALs
2024-11-08T23:43:26,400 INFO  [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C40483%2C1731109389343.meta:.meta(num 1731109393176)
2024-11-08T23:43:26,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741832_1008 (size=4131)
2024-11-08T23:43:26,405 DEBUG [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/oldWALs
2024-11-08T23:43:26,405 INFO  [RS:0;a15bf3ddec7f:40483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a15bf3ddec7f%2C40483%2C1731109389343:(num 1731109392285)
2024-11-08T23:43:26,405 DEBUG [RS:0;a15bf3ddec7f:40483 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T23:43:26,405 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T23:43:26,405 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T23:43:26,405 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.ChoreService(370): Chore service for: regionserver/a15bf3ddec7f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-08T23:43:26,405 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T23:43:26,405 INFO  [regionserver/a15bf3ddec7f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T23:43:26,406 INFO  [RS:0;a15bf3ddec7f:40483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40483
2024-11-08T23:43:26,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T23:43:26,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a15bf3ddec7f,40483,1731109389343
2024-11-08T23:43:26,410 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T23:43:26,411 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a15bf3ddec7f,40483,1731109389343]
2024-11-08T23:43:26,414 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a15bf3ddec7f,40483,1731109389343 already deleted, retry=false
2024-11-08T23:43:26,414 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a15bf3ddec7f,40483,1731109389343 expired; onlineServers=0
2024-11-08T23:43:26,414 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a15bf3ddec7f,46285,1731109388673' *****
2024-11-08T23:43:26,414 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-08T23:43:26,415 INFO  [M:0;a15bf3ddec7f:46285 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T23:43:26,415 INFO  [M:0;a15bf3ddec7f:46285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T23:43:26,415 DEBUG [M:0;a15bf3ddec7f:46285 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-08T23:43:26,415 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-08T23:43:26,415 DEBUG [M:0;a15bf3ddec7f:46285 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-08T23:43:26,415 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109391926 {}] cleaner.HFileCleaner(306): Exit Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.large.0-1731109391926,5,FailOnTimeoutGroup]
2024-11-08T23:43:26,415 DEBUG [master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109391927 {}] cleaner.HFileCleaner(306): Exit Thread[master/a15bf3ddec7f:0:becomeActiveMaster-HFileCleaner.small.0-1731109391927,5,FailOnTimeoutGroup]
2024-11-08T23:43:26,415 INFO  [M:0;a15bf3ddec7f:46285 {}] hbase.ChoreService(370): Chore service for: master/a15bf3ddec7f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-08T23:43:26,415 INFO  [M:0;a15bf3ddec7f:46285 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T23:43:26,415 DEBUG [M:0;a15bf3ddec7f:46285 {}] master.HMaster(1795): Stopping service threads
2024-11-08T23:43:26,415 INFO  [M:0;a15bf3ddec7f:46285 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-08T23:43:26,416 INFO  [M:0;a15bf3ddec7f:46285 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T23:43:26,416 ERROR [M:0;a15bf3ddec7f:46285 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[HFileArchiver-3,5,PEWorkerGroup]
    Thread[HFileArchiver-4,5,PEWorkerGroup]
    Thread[HFileArchiver-5,5,PEWorkerGroup]
    Thread[HFileArchiver-6,5,PEWorkerGroup]
2024-11-08T23:43:26,416 INFO  [M:0;a15bf3ddec7f:46285 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T23:43:26,416 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-08T23:43:26,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T23:43:26,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T23:43:26,417 DEBUG [M:0;a15bf3ddec7f:46285 {}] zookeeper.ZKUtil(347): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-08T23:43:26,417 WARN  [M:0;a15bf3ddec7f:46285 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-08T23:43:26,417 INFO  [M:0;a15bf3ddec7f:46285 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/.lastflushedseqids
2024-11-08T23:43:26,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741848_1024 (size=72)
2024-11-08T23:43:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T23:43:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40483-0x100f9f6e8bb0001, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T23:43:26,514 INFO  [RS:0;a15bf3ddec7f:40483 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T23:43:26,514 INFO  [RS:0;a15bf3ddec7f:40483 {}] regionserver.HRegionServer(1031): Exiting; stopping=a15bf3ddec7f,40483,1731109389343; zookeeper connection closed.
2024-11-08T23:43:26,515 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e2b6f30 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e2b6f30
2024-11-08T23:43:26,515 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-08T23:43:26,830 INFO  [M:0;a15bf3ddec7f:46285 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T23:43:26,830 INFO  [M:0;a15bf3ddec7f:46285 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-08T23:43:26,830 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-08T23:43:26,830 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:26,830 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:26,830 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-08T23:43:26,830 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:26,830 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=149.11 KB heapSize=178.91 KB
2024-11-08T23:43:26,849 DEBUG [M:0;a15bf3ddec7f:46285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8726447efd4e44be92795759c1216f62 is 82, key is hbase:meta,,1/info:regioninfo/1731109393261/Put/seqid=0
2024-11-08T23:43:26,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741849_1025 (size=5672)
2024-11-08T23:43:27,255 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8726447efd4e44be92795759c1216f62
2024-11-08T23:43:27,279 DEBUG [M:0;a15bf3ddec7f:46285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/50bebb6284e643f8981d35490b72da3a is 1206, key is \x00\x00\x00\x00\x00\x00\x00\x16/proc:d/1731109398153/Put/seqid=0
2024-11-08T23:43:27,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741850_1026 (size=11519)
2024-11-08T23:43:27,684 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148.55 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/50bebb6284e643f8981d35490b72da3a
2024-11-08T23:43:27,708 DEBUG [M:0;a15bf3ddec7f:46285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b589b7ea2bd846e1a7cedd295b5cbe5b is 69, key is a15bf3ddec7f,40483,1731109389343/rs:state/1731109392056/Put/seqid=0
2024-11-08T23:43:27,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741851_1027 (size=5156)
2024-11-08T23:43:28,114 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b589b7ea2bd846e1a7cedd295b5cbe5b
2024-11-08T23:43:28,122 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8726447efd4e44be92795759c1216f62 as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8726447efd4e44be92795759c1216f62
2024-11-08T23:43:28,128 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8726447efd4e44be92795759c1216f62, entries=8, sequenceid=328, filesize=5.5 K
2024-11-08T23:43:28,129 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/50bebb6284e643f8981d35490b72da3a as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/50bebb6284e643f8981d35490b72da3a
2024-11-08T23:43:28,136 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/50bebb6284e643f8981d35490b72da3a, entries=39, sequenceid=328, filesize=11.2 K
2024-11-08T23:43:28,137 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b589b7ea2bd846e1a7cedd295b5cbe5b as hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b589b7ea2bd846e1a7cedd295b5cbe5b
2024-11-08T23:43:28,143 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38449/user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b589b7ea2bd846e1a7cedd295b5cbe5b, entries=1, sequenceid=328, filesize=5.0 K
2024-11-08T23:43:28,144 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.11 KB/152686, heapSize ~178.61 KB/182896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=328, compaction requested=false
2024-11-08T23:43:28,146 INFO  [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T23:43:28,146 DEBUG [M:0;a15bf3ddec7f:46285 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731109406830Disabling compacts and flushes for region at 1731109406830Disabling writes for close at 1731109406830Obtaining lock to block concurrent updates at 1731109406830Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731109406830Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=152686, getHeapSize=183136, getOffHeapSize=0, getCellsCount=389 at 1731109406831 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731109406832 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731109406832Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731109406848 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731109406848Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731109407261 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731109407278 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731109407278Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731109407692 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731109407707 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731109407707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3386df72: reopening flushed file at 1731109408121 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e053c: reopening flushed file at 1731109408128 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6b0bfe: reopening flushed file at 1731109408136 (+8 ms)Finished flush of dataSize ~149.11 KB/152686, heapSize ~178.61 KB/182896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1314ms, sequenceid=328, compaction requested=false at 1731109408144 (+8 ms)Writing region close event to WAL at 1731109408146 (+2 ms)Closed at 1731109408146
2024-11-08T23:43:28,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741830_1006 (size=174669)
2024-11-08T23:43:28,149 WARN  [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e255d5c6-4af9-b387-75f9-4e99aa22cbf1/MasterData/WALs/a15bf3ddec7f,46285,1731109388673/a15bf3ddec7f%2C46285%2C1731109388673.1731109391385 not finished, retry = 0
2024-11-08T23:43:28,251 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T23:43:28,251 INFO  [M:0;a15bf3ddec7f:46285 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-08T23:43:28,251 INFO  [M:0;a15bf3ddec7f:46285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46285
2024-11-08T23:43:28,251 INFO  [M:0;a15bf3ddec7f:46285 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T23:43:28,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T23:43:28,353 INFO  [M:0;a15bf3ddec7f:46285 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T23:43:28,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46285-0x100f9f6e8bb0000, quorum=127.0.0.1:64298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T23:43:28,358 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18508316{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T23:43:28,360 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@d3378eb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T23:43:28,360 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T23:43:28,361 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@464c0fed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T23:43:28,361 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f4f11e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/hadoop.log.dir/,STOPPED}
2024-11-08T23:43:28,363 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T23:43:28,363 WARN  [BP-1559241834-172.17.0.2-1731109385919 heartbeating to localhost/127.0.0.1:38449 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T23:43:28,363 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T23:43:28,363 WARN  [BP-1559241834-172.17.0.2-1731109385919 heartbeating to localhost/127.0.0.1:38449 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1559241834-172.17.0.2-1731109385919 (Datanode Uuid 88a7e4e5-1681-48b0-866e-41fa7dc5e539) service to localhost/127.0.0.1:38449
2024-11-08T23:43:28,365 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/data/data1/current/BP-1559241834-172.17.0.2-1731109385919 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T23:43:28,365 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/cluster_fecf0386-c931-584e-8a3f-8172686cdd08/data/data2/current/BP-1559241834-172.17.0.2-1731109385919 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T23:43:28,365 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T23:43:28,373 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34d8ada7{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T23:43:28,374 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6067377f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T23:43:28,374 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T23:43:28,374 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f35049c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T23:43:28,374 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a627e45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7588fe24-2f2b-a55f-9da7-b953b170e009/hadoop.log.dir/,STOPPED}
2024-11-08T23:43:28,381 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-08T23:43:28,397 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down