2024-11-15 00:43:31,682 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 2024-11-15 00:43:31,697 main DEBUG Took 0.012502 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 00:43:31,697 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 00:43:31,698 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 00:43:31,699 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 00:43:31,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,709 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 00:43:31,722 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,726 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,728 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,729 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,730 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,731 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,732 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,734 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,736 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:43:31,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,738 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 00:43:31,740 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:43:31,741 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 00:43:31,744 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 00:43:31,744 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 00:43:31,746 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 00:43:31,746 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 00:43:31,757 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 00:43:31,760 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 00:43:31,763 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 00:43:31,763 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 00:43:31,764 main DEBUG createAppenders(={Console}) 2024-11-15 00:43:31,765 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 initialized 2024-11-15 00:43:31,766 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 2024-11-15 00:43:31,766 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 OK. 2024-11-15 00:43:31,767 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 00:43:31,767 main DEBUG OutputStream closed 2024-11-15 00:43:31,768 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 00:43:31,768 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 00:43:31,769 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4803b726 OK 2024-11-15 00:43:31,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 00:43:31,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 00:43:31,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 00:43:31,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 00:43:31,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 00:43:31,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 00:43:31,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 00:43:31,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 00:43:31,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 00:43:31,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 00:43:31,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 00:43:31,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 00:43:31,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 00:43:31,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 00:43:31,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 00:43:31,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 00:43:31,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 00:43:31,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 00:43:31,875 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 00:43:31,875 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null 2024-11-15 00:43:31,876 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 00:43:31,876 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK. 2024-11-15T00:43:32,169 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564 2024-11-15 00:43:32,171 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 00:43:32,172 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T00:43:32,180 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure timeout: 13 mins 2024-11-15T00:43:32,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T00:43:32,215 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f, deleteOnExit=true 2024-11-15T00:43:32,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T00:43:32,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/test.cache.data in system properties and HBase conf 2024-11-15T00:43:32,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T00:43:32,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/hadoop.log.dir in system properties and HBase conf 2024-11-15T00:43:32,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T00:43:32,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T00:43:32,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T00:43:32,337 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T00:43:32,449 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T00:43:32,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T00:43:32,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T00:43:32,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T00:43:32,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T00:43:32,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T00:43:32,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T00:43:32,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T00:43:32,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T00:43:32,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T00:43:32,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/nfs.dump.dir in system properties and HBase conf 2024-11-15T00:43:32,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/java.io.tmpdir in system properties and HBase conf 2024-11-15T00:43:32,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T00:43:32,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T00:43:32,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T00:43:33,645 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T00:43:33,740 INFO [Time-limited test {}] log.Log(170): Logging initialized @2766ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T00:43:33,825 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:43:33,897 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T00:43:33,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T00:43:33,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T00:43:33,918 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T00:43:33,930 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:43:33,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/hadoop.log.dir/,AVAILABLE} 2024-11-15T00:43:33,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T00:43:34,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27b19a13{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/java.io.tmpdir/jetty-localhost-46673-hadoop-hdfs-3_4_1-tests_jar-_-any-17464382111633146867/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T00:43:34,148 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:46673} 2024-11-15T00:43:34,148 INFO [Time-limited test {}] server.Server(415): Started @3175ms 2024-11-15T00:43:34,893 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:43:34,901 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T00:43:34,903 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T00:43:34,904 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T00:43:34,904 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T00:43:34,905 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/hadoop.log.dir/,AVAILABLE} 2024-11-15T00:43:34,906 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T00:43:35,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@156ffeaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/java.io.tmpdir/jetty-localhost-45509-hadoop-hdfs-3_4_1-tests_jar-_-any-14023113667849681920/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T00:43:35,012 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:45509} 2024-11-15T00:43:35,012 INFO [Time-limited test {}] server.Server(415): Started @4039ms 2024-11-15T00:43:35,065 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T00:43:36,215 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/data/data1/current/BP-147362170-172.17.0.3-1731631413147/current, will proceed with Du for space computation calculation, 2024-11-15T00:43:36,215 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/data/data2/current/BP-147362170-172.17.0.3-1731631413147/current, will proceed with Du for space computation calculation, 2024-11-15T00:43:36,279 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T00:43:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x580348de33f6ca17 with lease ID 0xd9303a1edd0dc434: Processing first storage report for DS-977703c7-3d43-464b-bf6e-2ee1d994d8c1 from datanode DatanodeRegistration(127.0.0.1:44695, datanodeUuid=2e9bf9a5-71c5-4405-bc8a-d74a59c9044b, infoPort=33623, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=2144722417;c=1731631413147) 2024-11-15T00:43:36,372 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x580348de33f6ca17 with lease ID 0xd9303a1edd0dc434: from storage DS-977703c7-3d43-464b-bf6e-2ee1d994d8c1 node DatanodeRegistration(127.0.0.1:44695, datanodeUuid=2e9bf9a5-71c5-4405-bc8a-d74a59c9044b, infoPort=33623, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=2144722417;c=1731631413147), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T00:43:36,372 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x580348de33f6ca17 with lease ID 0xd9303a1edd0dc434: Processing first storage report for DS-2ecc11ff-b28a-4ef4-95f7-2c4d6bf0ec71 from datanode DatanodeRegistration(127.0.0.1:44695, datanodeUuid=2e9bf9a5-71c5-4405-bc8a-d74a59c9044b, infoPort=33623, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=2144722417;c=1731631413147) 2024-11-15T00:43:36,373 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x580348de33f6ca17 with lease ID 0xd9303a1edd0dc434: from storage DS-2ecc11ff-b28a-4ef4-95f7-2c4d6bf0ec71 node DatanodeRegistration(127.0.0.1:44695, datanodeUuid=2e9bf9a5-71c5-4405-bc8a-d74a59c9044b, infoPort=33623, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=2144722417;c=1731631413147), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T00:43:36,418 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564 2024-11-15T00:43:36,528 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/zookeeper_0, clientPort=63272, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T00:43:36,541 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63272 2024-11-15T00:43:36,552 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:36,555 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:36,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741825_1001 (size=7) 2024-11-15T00:43:37,309 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 with version=8 2024-11-15T00:43:37,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/hbase-staging 2024-11-15T00:43:37,424 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T00:43:37,638 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fb2f6cb7846a:0 server-side Connection retries=45 2024-11-15T00:43:37,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:43:37,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T00:43:37,652 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T00:43:37,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:43:37,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T00:43:37,797 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T00:43:37,853 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T00:43:37,862 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T00:43:37,866 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T00:43:37,891 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4891 (auto-detected) 2024-11-15T00:43:37,892 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T00:43:37,910 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41375 2024-11-15T00:43:37,932 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41375 connecting to ZooKeeper ensemble=127.0.0.1:63272 2024-11-15T00:43:38,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413750x0, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T00:43:38,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41375-0x1013bf2805d0000 connected 2024-11-15T00:43:38,151 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:38,156 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:38,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:43:38,171 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6, hbase.cluster.distributed=false 2024-11-15T00:43:38,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T00:43:38,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41375 2024-11-15T00:43:38,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41375 2024-11-15T00:43:38,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41375 2024-11-15T00:43:38,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41375 2024-11-15T00:43:38,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41375 2024-11-15T00:43:38,297 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fb2f6cb7846a:0 server-side Connection retries=45 2024-11-15T00:43:38,299 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:43:38,299 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T00:43:38,299 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T00:43:38,299 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:43:38,300 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T00:43:38,302 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T00:43:38,305 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T00:43:38,306 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46669 2024-11-15T00:43:38,308 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46669 connecting to ZooKeeper ensemble=127.0.0.1:63272 2024-11-15T00:43:38,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:38,312 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:38,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466690x0, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T00:43:38,328 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466690x0, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:43:38,328 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46669-0x1013bf2805d0001 connected 2024-11-15T00:43:38,332 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T00:43:38,340 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T00:43:38,342 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T00:43:38,348 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T00:43:38,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46669 2024-11-15T00:43:38,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46669 2024-11-15T00:43:38,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46669 2024-11-15T00:43:38,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46669 2024-11-15T00:43:38,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46669 2024-11-15T00:43:38,370 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fb2f6cb7846a:41375 2024-11-15T00:43:38,370 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:38,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:43:38,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:43:38,382 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:38,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T00:43:38,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:38,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:38,472 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T00:43:38,475 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fb2f6cb7846a,41375,1731631417476 from backup master directory 2024-11-15T00:43:38,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:43:38,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:38,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:43:38,532 WARN [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T00:43:38,533 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:38,535 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T00:43:38,537 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T00:43:38,592 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/hbase.id] with ID: 382aa529-330b-48b3-ab39-11185352d1f6 2024-11-15T00:43:38,592 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/.tmp/hbase.id 2024-11-15T00:43:38,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741826_1002 (size=42) 2024-11-15T00:43:39,006 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/.tmp/hbase.id]:[hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/hbase.id] 2024-11-15T00:43:39,059 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:43:39,065 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T00:43:39,091 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-15T00:43:39,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:39,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:39,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741827_1003 (size=196) 2024-11-15T00:43:39,591 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:43:39,593 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T00:43:39,608 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:43:39,611 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:43:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741828_1004 (size=1189) 2024-11-15T00:43:39,664 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store 2024-11-15T00:43:39,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741829_1005 (size=34) 2024-11-15T00:43:39,702 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T00:43:39,710 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:39,712 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T00:43:39,713 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:39,713 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:39,715 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T00:43:39,716 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:39,716 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:39,718 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731631419712Disabling compacts and flushes for region at 1731631419712Disabling writes for close at 1731631419715 (+3 ms)Writing region close event to WAL at 1731631419716 (+1 ms)Closed at 1731631419716 2024-11-15T00:43:39,723 WARN [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/.initializing 2024-11-15T00:43:39,723 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/WALs/fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:39,734 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T00:43:39,755 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C41375%2C1731631417476, suffix=, logDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/WALs/fb2f6cb7846a,41375,1731631417476, archiveDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/oldWALs, maxLogs=10 2024-11-15T00:43:39,786 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/WALs/fb2f6cb7846a,41375,1731631417476/fb2f6cb7846a%2C41375%2C1731631417476.1731631419762, exclude list is [], retry=0 2024-11-15T00:43:39,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44695,DS-977703c7-3d43-464b-bf6e-2ee1d994d8c1,DISK] 2024-11-15T00:43:39,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T00:43:39,860 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/WALs/fb2f6cb7846a,41375,1731631417476/fb2f6cb7846a%2C41375%2C1731631417476.1731631419762 2024-11-15T00:43:39,861 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-15T00:43:39,862 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:43:39,863 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:39,866 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,867 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,945 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T00:43:39,950 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:39,953 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:39,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T00:43:39,964 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:39,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:39,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T00:43:39,976 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:39,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:39,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T00:43:39,988 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:39,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:39,990 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,996 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:39,999 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:40,014 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:40,021 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:40,028 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T00:43:40,033 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:43:40,046 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:40,049 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71178792, jitterRate=0.060646653175354004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T00:43:40,065 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731631419880Initializing all the Stores at 1731631419882 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631419883 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631419884 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631419885 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631419885Cleaning up temporary data from old regions at 1731631420021 (+136 ms)Region opened successfully at 1731631420065 (+44 ms) 2024-11-15T00:43:40,072 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T00:43:40,120 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6535d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fb2f6cb7846a/172.17.0.3:0 2024-11-15T00:43:40,158 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T00:43:40,169 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:40,169 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:40,172 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:40,173 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:40,179 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T00:43:40,179 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T00:43:40,212 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T00:43:40,225 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T00:43:40,232 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T00:43:40,235 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T00:43:40,237 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T00:43:40,253 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T00:43:40,256 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T00:43:40,261 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T00:43:40,274 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T00:43:40,276 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T00:43:40,285 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T00:43:40,309 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T00:43:40,316 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T00:43:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T00:43:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T00:43:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:40,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:40,330 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fb2f6cb7846a,41375,1731631417476, sessionid=0x1013bf2805d0000, setting cluster-up flag (Was=false) 2024-11-15T00:43:40,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:40,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:40,390 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T00:43:40,393 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:40,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:40,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:40,464 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T00:43:40,466 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:40,474 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T00:43:40,563 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(746): ClusterId : 382aa529-330b-48b3-ab39-11185352d1f6 2024-11-15T00:43:40,567 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T00:43:40,576 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:40,583 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T00:43:40,583 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T00:43:40,589 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T00:43:40,591 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T00:43:40,592 DEBUG [RS:0;fb2f6cb7846a:46669 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631b7ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fb2f6cb7846a/172.17.0.3:0 2024-11-15T00:43:40,597 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T00:43:40,605 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fb2f6cb7846a,41375,1731631417476 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T00:43:40,613 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fb2f6cb7846a:46669 2024-11-15T00:43:40,615 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:43:40,616 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:43:40,616 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:43:40,616 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:43:40,617 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fb2f6cb7846a:0, corePoolSize=10, maxPoolSize=10 2024-11-15T00:43:40,617 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T00:43:40,617 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,617 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T00:43:40,617 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T00:43:40,617 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T00:43:40,617 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,620 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(2659): reportForDuty to master=fb2f6cb7846a,41375,1731631417476 with port=46669, startcode=1731631418265 2024-11-15T00:43:40,628 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631450628 2024-11-15T00:43:40,630 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T00:43:40,631 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T00:43:40,636 DEBUG [RS:0;fb2f6cb7846a:46669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T00:43:40,637 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T00:43:40,638 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T00:43:40,638 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T00:43:40,639 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T00:43:40,639 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:40,640 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T00:43:40,646 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:40,647 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T00:43:40,651 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,659 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T00:43:40,662 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T00:43:40,662 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T00:43:40,669 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T00:43:40,669 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T00:43:40,673 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631420671,5,FailOnTimeoutGroup] 2024-11-15T00:43:40,677 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631420673,5,FailOnTimeoutGroup] 2024-11-15T00:43:40,678 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,679 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T00:43:40,681 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,682 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741831_1007 (size=1321) 2024-11-15T00:43:40,698 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38423, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T00:43:40,702 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T00:43:40,703 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:40,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:40,716 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] master.ServerManager(517): Registering regionserver=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:40,736 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:40,736 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34361 2024-11-15T00:43:40,736 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T00:43:40,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741832_1008 (size=32) 2024-11-15T00:43:40,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T00:43:40,774 DEBUG [RS:0;fb2f6cb7846a:46669 {}] zookeeper.ZKUtil(111): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:40,775 WARN [RS:0;fb2f6cb7846a:46669 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T00:43:40,775 INFO [RS:0;fb2f6cb7846a:46669 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:43:40,775 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:40,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fb2f6cb7846a,46669,1731631418265] 2024-11-15T00:43:40,815 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T00:43:40,838 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T00:43:40,845 INFO [RS:0;fb2f6cb7846a:46669 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T00:43:40,846 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,849 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T00:43:40,858 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T00:43:40,861 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,862 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,862 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,862 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,863 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,863 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,863 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fb2f6cb7846a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T00:43:40,863 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,864 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,865 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,865 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,866 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,866 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:43:40,866 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fb2f6cb7846a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T00:43:40,866 DEBUG [RS:0;fb2f6cb7846a:46669 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T00:43:40,872 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,875 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,875 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,875 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,875 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,875 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46669,1731631418265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T00:43:40,907 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T00:43:40,909 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46669,1731631418265-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,910 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,911 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.Replication(171): fb2f6cb7846a,46669,1731631418265 started 2024-11-15T00:43:40,932 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:40,932 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1482): Serving as fb2f6cb7846a,46669,1731631418265, RpcServer on fb2f6cb7846a/172.17.0.3:46669, sessionid=0x1013bf2805d0001 2024-11-15T00:43:40,933 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T00:43:40,933 DEBUG [RS:0;fb2f6cb7846a:46669 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:40,933 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fb2f6cb7846a,46669,1731631418265' 2024-11-15T00:43:40,934 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T00:43:40,937 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T00:43:40,937 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T00:43:40,938 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T00:43:40,938 DEBUG [RS:0;fb2f6cb7846a:46669 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:40,938 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fb2f6cb7846a,46669,1731631418265' 2024-11-15T00:43:40,938 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T00:43:40,939 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T00:43:40,940 DEBUG [RS:0;fb2f6cb7846a:46669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T00:43:40,940 INFO [RS:0;fb2f6cb7846a:46669 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T00:43:40,940 INFO [RS:0;fb2f6cb7846a:46669 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T00:43:41,047 INFO [RS:0;fb2f6cb7846a:46669 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T00:43:41,052 INFO [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C46669%2C1731631418265, suffix=, logDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265, archiveDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/oldWALs, maxLogs=32 2024-11-15T00:43:41,070 DEBUG [RS:0;fb2f6cb7846a:46669 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265/fb2f6cb7846a%2C46669%2C1731631418265.1731631421055, exclude list is [], retry=0 2024-11-15T00:43:41,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44695,DS-977703c7-3d43-464b-bf6e-2ee1d994d8c1,DISK] 2024-11-15T00:43:41,081 INFO [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265/fb2f6cb7846a%2C46669%2C1731631418265.1731631421055 2024-11-15T00:43:41,082 DEBUG [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-15T00:43:41,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:41,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T00:43:41,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T00:43:41,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T00:43:41,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T00:43:41,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T00:43:41,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T00:43:41,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T00:43:41,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T00:43:41,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T00:43:41,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740 2024-11-15T00:43:41,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740 2024-11-15T00:43:41,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T00:43:41,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T00:43:41,170 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T00:43:41,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T00:43:41,177 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:41,178 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62833677, jitterRate=-0.06370525062084198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T00:43:41,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731631421140Initializing all the Stores at 1731631421142 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631421142Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631421142Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631421142Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631421142Cleaning up temporary data from old regions at 1731631421169 (+27 ms)Region opened successfully at 1731631421181 (+12 ms) 2024-11-15T00:43:41,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T00:43:41,182 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T00:43:41,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T00:43:41,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T00:43:41,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T00:43:41,184 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T00:43:41,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731631421182Disabling compacts and flushes for region at 1731631421182Disabling writes for close at 1731631421182Writing region close event to WAL at 1731631421183 (+1 ms)Closed at 1731631421184 (+1 ms) 2024-11-15T00:43:41,188 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:41,189 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T00:43:41,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T00:43:41,210 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T00:43:41,214 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T00:43:41,368 DEBUG [fb2f6cb7846a:41375 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T00:43:41,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:41,389 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPENING 2024-11-15T00:43:41,524 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T00:43:41,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:41,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:41,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:41,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:41,540 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T00:43:41,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:41,720 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T00:43:41,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T00:43:41,742 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T00:43:41,742 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:43:41,743 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T00:43:41,747 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C46669%2C1731631418265.meta, suffix=.meta, logDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265, archiveDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/oldWALs, maxLogs=32 2024-11-15T00:43:41,769 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265/fb2f6cb7846a%2C46669%2C1731631418265.meta.1731631421749.meta, exclude list is [], retry=0 2024-11-15T00:43:41,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44695,DS-977703c7-3d43-464b-bf6e-2ee1d994d8c1,DISK] 2024-11-15T00:43:41,780 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265/fb2f6cb7846a%2C46669%2C1731631418265.meta.1731631421749.meta 2024-11-15T00:43:41,781 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33623:33623)] 2024-11-15T00:43:41,781 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:43:41,783 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T00:43:41,786 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T00:43:41,790 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T00:43:41,794 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T00:43:41,795 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:41,795 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T00:43:41,795 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T00:43:41,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T00:43:41,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T00:43:41,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T00:43:41,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T00:43:41,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T00:43:41,808 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T00:43:41,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T00:43:41,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T00:43:41,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:41,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:43:41,813 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T00:43:41,815 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740 2024-11-15T00:43:41,818 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740 2024-11-15T00:43:41,820 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T00:43:41,821 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T00:43:41,822 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T00:43:41,825 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T00:43:41,828 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71290360, jitterRate=0.0623091459274292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T00:43:41,828 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T00:43:41,830 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731631421796Writing region info on filesystem at 1731631421796Initializing all the Stores at 1731631421798 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631421798Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631421799 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631421799Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631421799Cleaning up temporary data from old regions at 1731631421821 (+22 ms)Running coprocessor post-open hooks at 1731631421828 (+7 ms)Region opened successfully at 1731631421829 (+1 ms) 2024-11-15T00:43:41,836 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731631421712 2024-11-15T00:43:41,848 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T00:43:41,848 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T00:43:41,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:41,853 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:41,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:41,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:41,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:41,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:41,897 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:41,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T00:43:41,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,46669,1731631418265 in 355 msec 2024-11-15T00:43:41,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T00:43:41,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 712 msec 2024-11-15T00:43:41,918 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:41,918 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T00:43:41,945 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:43:41,947 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,46669,1731631418265, seqNum=-1] 2024-11-15T00:43:41,977 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:43:41,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53133, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:43:42,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4860 sec 2024-11-15T00:43:42,003 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731631422003, completionTime=-1 2024-11-15T00:43:42,005 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T00:43:42,005 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:42,031 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:42,032 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631482031 2024-11-15T00:43:42,032 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631542032 2024-11-15T00:43:42,032 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-15T00:43:42,034 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,41375,1731631417476-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,035 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,41375,1731631417476-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,035 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,41375,1731631417476-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,037 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fb2f6cb7846a:41375, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,037 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,038 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,044 DEBUG [master/fb2f6cb7846a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T00:43:42,069 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.536sec 2024-11-15T00:43:42,070 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T00:43:42,071 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T00:43:42,072 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T00:43:42,073 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T00:43:42,073 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T00:43:42,074 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,41375,1731631417476-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T00:43:42,074 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,41375,1731631417476-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T00:43:42,087 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T00:43:42,088 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T00:43:42,089 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,41375,1731631417476-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:43:42,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d7c978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:43:42,175 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T00:43:42,175 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T00:43:42,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,41375,-1 for getting cluster id 2024-11-15T00:43:42,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:43:42,190 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '382aa529-330b-48b3-ab39-11185352d1f6' 2024-11-15T00:43:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:43:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "382aa529-330b-48b3-ab39-11185352d1f6" 2024-11-15T00:43:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8df8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:43:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,41375,-1] 2024-11-15T00:43:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:43:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:43:42,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:43:42,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df26bdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:43:42,208 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:43:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,46669,1731631418265, seqNum=-1] 2024-11-15T00:43:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:43:42,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:43:42,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:42,261 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTable Thread=205, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=2204 2024-11-15T00:43:42,266 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T00:43:42,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:42,280 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:43:42,283 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:42,287 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:43:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741835_1011 (size=639) 2024-11-15T00:43:42,705 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e1fef6076410796fcf56c873525c83f, NAME => 'testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741836_1012 (size=51) 2024-11-15T00:43:43,137 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:43,137 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1722): Closing 5e1fef6076410796fcf56c873525c83f, disabling compactions & flushes 2024-11-15T00:43:43,138 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,138 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,138 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. after waiting 0 ms 2024-11-15T00:43:43,138 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,138 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,138 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e1fef6076410796fcf56c873525c83f: Waiting for close lock at 1731631423137Disabling compacts and flushes for region at 1731631423137Disabling writes for close at 1731631423138 (+1 ms)Writing region close event to WAL at 1731631423138Closed at 1731631423138 2024-11-15T00:43:43,142 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:43:43,149 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731631423142"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631423142"}]},"ts":"1731631423142"} 2024-11-15T00:43:43,156 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:43:43,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:43:43,163 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631423159"}]},"ts":"1731631423159"} 2024-11-15T00:43:43,169 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLING in hbase:meta 2024-11-15T00:43:43,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, ASSIGN}] 2024-11-15T00:43:43,177 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, ASSIGN 2024-11-15T00:43:43,183 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,46669,1731631418265; forceNewPlan=false, retain=false 2024-11-15T00:43:43,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5e1fef6076410796fcf56c873525c83f, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:43,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, ASSIGN because future has completed 2024-11-15T00:43:43,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e1fef6076410796fcf56c873525c83f, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:43,511 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,511 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e1fef6076410796fcf56c873525c83f, NAME => 'testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:43:43,512 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTable 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,512 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:43,513 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,513 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,516 INFO [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,520 INFO [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e1fef6076410796fcf56c873525c83f columnFamilyName f1 2024-11-15T00:43:43,520 DEBUG [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:43,522 INFO [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] regionserver.HStore(327): Store=5e1fef6076410796fcf56c873525c83f/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:43,522 INFO [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,525 INFO [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e1fef6076410796fcf56c873525c83f columnFamilyName f2 2024-11-15T00:43:43,525 DEBUG [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:43,527 INFO [StoreOpener-5e1fef6076410796fcf56c873525c83f-1 {}] regionserver.HStore(327): Store=5e1fef6076410796fcf56c873525c83f/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:43,527 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,529 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,531 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,533 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,533 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,534 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTable descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T00:43:43,537 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,543 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:43,544 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5e1fef6076410796fcf56c873525c83f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68086539, jitterRate=0.014568492770195007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T00:43:43,545 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,547 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5e1fef6076410796fcf56c873525c83f: Running coprocessor pre-open hook at 1731631423513Writing region info on filesystem at 1731631423513Initializing all the Stores at 1731631423515 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631423515Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631423516 (+1 ms)Cleaning up temporary data from old regions at 1731631423533 (+17 ms)Running coprocessor post-open hooks at 1731631423545 (+12 ms)Region opened successfully at 1731631423547 (+2 ms) 2024-11-15T00:43:43,550 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f., pid=6, masterSystemTime=1731631423500 2024-11-15T00:43:43,555 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,555 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,557 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5e1fef6076410796fcf56c873525c83f, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:43,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e1fef6076410796fcf56c873525c83f, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:43,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T00:43:43,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e1fef6076410796fcf56c873525c83f, server=fb2f6cb7846a,46669,1731631418265 in 222 msec 2024-11-15T00:43:43,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T00:43:43,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, ASSIGN in 402 msec 2024-11-15T00:43:43,582 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:43:43,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631423582"}]},"ts":"1731631423582"} 2024-11-15T00:43:43,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLED in hbase:meta 2024-11-15T00:43:43,591 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:43:43,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable in 1.3230 sec 2024-11-15T00:43:43,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:43,796 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631423795"}]},"ts":"1731631423795"} 2024-11-15T00:43:43,801 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLING in hbase:meta 2024-11-15T00:43:43,801 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTable to state=DISABLING 2024-11-15T00:43:43,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTable}] 2024-11-15T00:43:43,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, UNASSIGN}] 2024-11-15T00:43:43,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, UNASSIGN 2024-11-15T00:43:43,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5e1fef6076410796fcf56c873525c83f, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:43,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, UNASSIGN because future has completed 2024-11-15T00:43:43,832 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:43,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e1fef6076410796fcf56c873525c83f, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:43,994 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:43,994 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:43:43,995 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 5e1fef6076410796fcf56c873525c83f, disabling compactions & flushes 2024-11-15T00:43:43,995 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,995 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:43,995 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. after waiting 0 ms 2024-11-15T00:43:43,995 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:44,017 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:43:44,019 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f. 2024-11-15T00:43:44,019 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 5e1fef6076410796fcf56c873525c83f: Waiting for close lock at 1731631423995Running coprocessor pre-close hooks at 1731631423995Disabling compacts and flushes for region at 1731631423995Disabling writes for close at 1731631423995Writing region close event to WAL at 1731631423996 (+1 ms)Running coprocessor post-close hooks at 1731631424018 (+22 ms)Closed at 1731631424019 (+1 ms) 2024-11-15T00:43:44,028 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:44,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5e1fef6076410796fcf56c873525c83f, regionState=CLOSED 2024-11-15T00:43:44,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e1fef6076410796fcf56c873525c83f, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:44,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T00:43:44,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e1fef6076410796fcf56c873525c83f, server=fb2f6cb7846a,46669,1731631418265 in 204 msec 2024-11-15T00:43:44,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-15T00:43:44,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=5e1fef6076410796fcf56c873525c83f, UNASSIGN in 227 msec 2024-11-15T00:43:44,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T00:43:44,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTable in 249 msec 2024-11-15T00:43:44,067 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631424066"}]},"ts":"1731631424066"} 2024-11-15T00:43:44,072 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLED in hbase:meta 2024-11-15T00:43:44,072 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTable to state=DISABLED 2024-11-15T00:43:44,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable in 294 msec 2024-11-15T00:43:44,288 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T00:43:44,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:43:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c32b5a2 2024-11-15T00:43:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:43:44,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:43:44,329 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTable 2024-11-15T00:43:44,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable testDisableTable 2024-11-15T00:43:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] ipc.CallRunner(138): callId: 2 service: MasterService methodName: DisableTable size: 77 connection: 172.17.0.3:52328 deadline: 1731631484333, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable 2024-11-15T00:43:44,364 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTable already disabled, so just deleting it. 2024-11-15T00:43:44,364 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTable failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:43:44,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete testDisableTable 2024-11-15T00:43:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] procedure2.ProcedureExecutor(1139): Stored pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:44,394 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:44,399 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=12, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-15T00:43:44,412 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:44,427 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/f1, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/f2, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/recovered.edits] 2024-11-15T00:43:44,447 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/recovered.edits/4.seqid to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/archive/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f/recovered.edits/4.seqid 2024-11-15T00:43:44,449 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTable/5e1fef6076410796fcf56c873525c83f 2024-11-15T00:43:44,450 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTable regions 2024-11-15T00:43:44,480 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=12, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:44,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46669 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T00:43:44,497 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTable from hbase:meta 2024-11-15T00:43:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-15T00:43:44,569 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTable' descriptor. 2024-11-15T00:43:44,573 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=12, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:44,574 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTable' from region states. 2024-11-15T00:43:44,574 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631424574"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:44,581 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:43:44,582 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5e1fef6076410796fcf56c873525c83f, NAME => 'testDisableTable,,1731631422268.5e1fef6076410796fcf56c873525c83f.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:43:44,582 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTable' as deleted. 2024-11-15T00:43:44,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631424582"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:44,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTable state from META 2024-11-15T00:43:44,591 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=12, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:44,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable in 206 msec 2024-11-15T00:43:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-15T00:43:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTable 2024-11-15T00:43:44,725 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTable completed 2024-11-15T00:43:44,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTable Thread=215 (was 205) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_62144211_22 at /127.0.0.1:50654 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 431) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1969 (was 2204) 2024-11-15T00:43:44,748 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=215, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=1969 2024-11-15T00:43:44,748 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T00:43:44,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:44,753 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:43:44,754 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:44,756 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:43:44,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741837_1013 (size=652) 2024-11-15T00:43:44,772 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a06f769724430ccd8d2e0f43c9e1c95b, NAME => 'testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTableMultipleTimes', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:44,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741838_1014 (size=64) 2024-11-15T00:43:44,788 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:44,789 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1722): Closing a06f769724430ccd8d2e0f43c9e1c95b, disabling compactions & flushes 2024-11-15T00:43:44,789 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:44,789 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:44,789 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. after waiting 0 ms 2024-11-15T00:43:44,789 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:44,789 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:44,789 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1676): Region close journal for a06f769724430ccd8d2e0f43c9e1c95b: Waiting for close lock at 1731631424789Disabling compacts and flushes for region at 1731631424789Disabling writes for close at 1731631424789Writing region close event to WAL at 1731631424789Closed at 1731631424789 2024-11-15T00:43:44,791 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:43:44,792 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731631424792"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631424792"}]},"ts":"1731631424792"} 2024-11-15T00:43:44,797 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:43:44,800 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:43:44,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631424800"}]},"ts":"1731631424800"} 2024-11-15T00:43:44,803 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLING in hbase:meta 2024-11-15T00:43:44,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, ASSIGN}] 2024-11-15T00:43:44,808 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, ASSIGN 2024-11-15T00:43:44,810 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,46669,1731631418265; forceNewPlan=false, retain=false 2024-11-15T00:43:44,961 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a06f769724430ccd8d2e0f43c9e1c95b, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:44,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, ASSIGN because future has completed 2024-11-15T00:43:44,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a06f769724430ccd8d2e0f43c9e1c95b, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:45,129 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,130 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => a06f769724430ccd8d2e0f43c9e1c95b, NAME => 'testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:43:45,131 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTableMultipleTimes a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,131 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:45,131 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,132 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,135 INFO [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,138 INFO [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a06f769724430ccd8d2e0f43c9e1c95b columnFamilyName f1 2024-11-15T00:43:45,138 DEBUG [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:45,139 INFO [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] regionserver.HStore(327): Store=a06f769724430ccd8d2e0f43c9e1c95b/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:45,139 INFO [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,142 INFO [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a06f769724430ccd8d2e0f43c9e1c95b columnFamilyName f2 2024-11-15T00:43:45,143 DEBUG [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:45,144 INFO [StoreOpener-a06f769724430ccd8d2e0f43c9e1c95b-1 {}] regionserver.HStore(327): Store=a06f769724430ccd8d2e0f43c9e1c95b/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:45,144 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,146 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,147 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,148 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,148 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,149 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTableMultipleTimes descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T00:43:45,151 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,155 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:45,155 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened a06f769724430ccd8d2e0f43c9e1c95b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62039788, jitterRate=-0.0755351185798645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T00:43:45,156 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,157 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for a06f769724430ccd8d2e0f43c9e1c95b: Running coprocessor pre-open hook at 1731631425132Writing region info on filesystem at 1731631425132Initializing all the Stores at 1731631425134 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631425134Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631425135 (+1 ms)Cleaning up temporary data from old regions at 1731631425148 (+13 ms)Running coprocessor post-open hooks at 1731631425156 (+8 ms)Region opened successfully at 1731631425156 2024-11-15T00:43:45,158 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b., pid=15, masterSystemTime=1731631425119 2024-11-15T00:43:45,161 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,161 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a06f769724430ccd8d2e0f43c9e1c95b, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:45,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a06f769724430ccd8d2e0f43c9e1c95b, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:45,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-15T00:43:45,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure a06f769724430ccd8d2e0f43c9e1c95b, server=fb2f6cb7846a,46669,1731631418265 in 202 msec 2024-11-15T00:43:45,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T00:43:45,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, ASSIGN in 368 msec 2024-11-15T00:43:45,177 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:43:45,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631425177"}]},"ts":"1731631425177"} 2024-11-15T00:43:45,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLED in hbase:meta 2024-11-15T00:43:45,182 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:43:45,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes in 434 msec 2024-11-15T00:43:45,254 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631425259"}]},"ts":"1731631425259"} 2024-11-15T00:43:45,265 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLING in hbase:meta 2024-11-15T00:43:45,265 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTableMultipleTimes to state=DISABLING 2024-11-15T00:43:45,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes}] 2024-11-15T00:43:45,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, UNASSIGN}] 2024-11-15T00:43:45,274 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, UNASSIGN 2024-11-15T00:43:45,275 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a06f769724430ccd8d2e0f43c9e1c95b, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:45,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, UNASSIGN because future has completed 2024-11-15T00:43:45,279 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:45,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure a06f769724430ccd8d2e0f43c9e1c95b, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:45,435 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(122): Close a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,436 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:43:45,436 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1722): Closing a06f769724430ccd8d2e0f43c9e1c95b, disabling compactions & flushes 2024-11-15T00:43:45,436 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,436 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,436 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. after waiting 0 ms 2024-11-15T00:43:45,436 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,443 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:43:45,444 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b. 2024-11-15T00:43:45,444 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1676): Region close journal for a06f769724430ccd8d2e0f43c9e1c95b: Waiting for close lock at 1731631425436Running coprocessor pre-close hooks at 1731631425436Disabling compacts and flushes for region at 1731631425436Disabling writes for close at 1731631425436Writing region close event to WAL at 1731631425437 (+1 ms)Running coprocessor post-close hooks at 1731631425444 (+7 ms)Closed at 1731631425444 2024-11-15T00:43:45,448 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(157): Closed a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=a06f769724430ccd8d2e0f43c9e1c95b, regionState=CLOSED 2024-11-15T00:43:45,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure a06f769724430ccd8d2e0f43c9e1c95b, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:45,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-15T00:43:45,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; CloseRegionProcedure a06f769724430ccd8d2e0f43c9e1c95b, server=fb2f6cb7846a,46669,1731631418265 in 175 msec 2024-11-15T00:43:45,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T00:43:45,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=a06f769724430ccd8d2e0f43c9e1c95b, UNASSIGN in 186 msec 2024-11-15T00:43:45,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-15T00:43:45,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes in 194 msec 2024-11-15T00:43:45,466 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631425466"}]},"ts":"1731631425466"} 2024-11-15T00:43:45,469 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLED in hbase:meta 2024-11-15T00:43:45,469 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTableMultipleTimes to state=DISABLED 2024-11-15T00:43:45,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes in 220 msec 2024-11-15T00:43:45,504 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(101): Disable failed with exception {}org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes 2024-11-15T00:43:45,505 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(114): Disable failed with expected exception {} org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:109) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:43:45,505 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(126): Disable failed with expected exception {} org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:64) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:120) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:43:45,506 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T00:43:45,508 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTableMultipleTimes 2024-11-15T00:43:45,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable testDisableTableMultipleTimes 2024-11-15T00:43:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] ipc.CallRunner(138): callId: 8 service: MasterService methodName: DisableTable size: 90 connection: 172.17.0.3:52328 deadline: 1731631485509, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes 2024-11-15T00:43:45,510 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTableMultipleTimes failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:43:45,511 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTableMultipleTimes already disabled, so just deleting it. 2024-11-15T00:43:45,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete testDisableTableMultipleTimes 2024-11-15T00:43:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,514 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:43:45,516 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,520 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,524 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/f1, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/f2, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/recovered.edits] 2024-11-15T00:43:45,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/recovered.edits/4.seqid to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/archive/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b/recovered.edits/4.seqid 2024-11-15T00:43:45,544 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testDisableTableMultipleTimes/a06f769724430ccd8d2e0f43c9e1c95b 2024-11-15T00:43:45,545 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTableMultipleTimes regions 2024-11-15T00:43:45,550 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,554 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTableMultipleTimes from hbase:meta 2024-11-15T00:43:45,565 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTableMultipleTimes' descriptor. 2024-11-15T00:43:45,567 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,567 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTableMultipleTimes' from region states. 2024-11-15T00:43:45,568 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631425567"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:45,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:43:45,571 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a06f769724430ccd8d2e0f43c9e1c95b, NAME => 'testDisableTableMultipleTimes,,1731631424748.a06f769724430ccd8d2e0f43c9e1c95b.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:43:45,571 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTableMultipleTimes' as deleted. 2024-11-15T00:43:45,571 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631425571"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:45,574 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTableMultipleTimes state from META 2024-11-15T00:43:45,576 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:45,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes in 65 msec 2024-11-15T00:43:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:43:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTableMultipleTimes 2024-11-15T00:43:45,627 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTableMultipleTimes completed 2024-11-15T00:43:45,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=220 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_62144211_22 at /127.0.0.1:50654 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 453), ProcessCount=11 (was 11), AvailableMemoryMB=3265 (was 1969) - AvailableMemoryMB LEAK? - 2024-11-15T00:43:45,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=220, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=3262 2024-11-15T00:43:45,649 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T00:43:45,653 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:45,655 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:43:45,656 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:45,658 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:43:45,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741839_1015 (size=653) 2024-11-15T00:43:46,077 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8735d34935ca3f779d7c7c3264ab101e, NAME => 'testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:46,077 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ddbd8b36972901bace84c2cfb9622f35, NAME => 'testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:46,079 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b17ecc70fa519806ead8c7410d465b33, NAME => 'testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:46,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741840_1016 (size=67) 2024-11-15T00:43:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741841_1017 (size=66) 2024-11-15T00:43:46,148 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:46,148 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing 8735d34935ca3f779d7c7c3264ab101e, disabling compactions & flushes 2024-11-15T00:43:46,148 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:46,148 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:46,149 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. after waiting 0 ms 2024-11-15T00:43:46,149 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:46,149 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:46,149 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8735d34935ca3f779d7c7c3264ab101e: Waiting for close lock at 1731631426148Disabling compacts and flushes for region at 1731631426148Disabling writes for close at 1731631426149 (+1 ms)Writing region close event to WAL at 1731631426149Closed at 1731631426149 2024-11-15T00:43:46,150 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 36002dcef3f83cc251444774b982f917, NAME => 'testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.', STARTKEY => 'c', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6 2024-11-15T00:43:46,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741842_1018 (size=67) 2024-11-15T00:43:46,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741843_1019 (size=66) 2024-11-15T00:43:46,544 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:46,544 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1722): Closing b17ecc70fa519806ead8c7410d465b33, disabling compactions & flushes 2024-11-15T00:43:46,544 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:46,544 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:46,544 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. after waiting 0 ms 2024-11-15T00:43:46,544 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:46,544 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:46,545 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1676): Region close journal for b17ecc70fa519806ead8c7410d465b33: Waiting for close lock at 1731631426544Disabling compacts and flushes for region at 1731631426544Disabling writes for close at 1731631426544Writing region close event to WAL at 1731631426544Closed at 1731631426544 2024-11-15T00:43:46,562 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:46,563 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing ddbd8b36972901bace84c2cfb9622f35, disabling compactions & flushes 2024-11-15T00:43:46,563 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,563 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,563 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. after waiting 0 ms 2024-11-15T00:43:46,563 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,563 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,563 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for ddbd8b36972901bace84c2cfb9622f35: Waiting for close lock at 1731631426562Disabling compacts and flushes for region at 1731631426562Disabling writes for close at 1731631426563 (+1 ms)Writing region close event to WAL at 1731631426563Closed at 1731631426563 2024-11-15T00:43:46,584 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:46,585 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing 36002dcef3f83cc251444774b982f917, disabling compactions & flushes 2024-11-15T00:43:46,585 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:46,585 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:46,585 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. after waiting 0 ms 2024-11-15T00:43:46,585 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:46,585 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:46,585 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for 36002dcef3f83cc251444774b982f917: Waiting for close lock at 1731631426585Disabling compacts and flushes for region at 1731631426585Disabling writes for close at 1731631426585Writing region close event to WAL at 1731631426585Closed at 1731631426585 2024-11-15T00:43:46,587 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:43:46,588 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731631426587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631426587"}]},"ts":"1731631426587"} 2024-11-15T00:43:46,588 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731631426587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631426587"}]},"ts":"1731631426587"} 2024-11-15T00:43:46,588 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731631426587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631426587"}]},"ts":"1731631426587"} 2024-11-15T00:43:46,588 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731631426587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631426587"}]},"ts":"1731631426587"} 2024-11-15T00:43:46,596 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 4 regions to meta. 2024-11-15T00:43:46,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:43:46,599 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631426599"}]},"ts":"1731631426599"} 2024-11-15T00:43:46,603 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLING in hbase:meta 2024-11-15T00:43:46,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, ASSIGN}, {pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, ASSIGN}, {pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, ASSIGN}, {pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, ASSIGN}] 2024-11-15T00:43:46,609 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, ASSIGN 2024-11-15T00:43:46,610 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,46669,1731631418265; forceNewPlan=false, retain=false 2024-11-15T00:43:46,613 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, ASSIGN 2024-11-15T00:43:46,614 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,46669,1731631418265; forceNewPlan=false, retain=false 2024-11-15T00:43:46,617 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, ASSIGN 2024-11-15T00:43:46,619 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,46669,1731631418265; forceNewPlan=false, retain=false 2024-11-15T00:43:46,621 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, ASSIGN 2024-11-15T00:43:46,629 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,46669,1731631418265; forceNewPlan=false, retain=false 2024-11-15T00:43:46,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:46,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:46,765 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:46,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, ASSIGN because future has completed 2024-11-15T00:43:46,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:46,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, ASSIGN because future has completed 2024-11-15T00:43:46,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, ASSIGN because future has completed 2024-11-15T00:43:46,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:46,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:46,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8735d34935ca3f779d7c7c3264ab101e, regionState=OPENING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:46,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, ASSIGN because future has completed 2024-11-15T00:43:46,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:46,929 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,930 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => ddbd8b36972901bace84c2cfb9622f35, NAME => 'testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.', STARTKEY => 'a', ENDKEY => 'b'} 2024-11-15T00:43:46,930 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,930 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:46,930 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,930 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,933 INFO [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,937 INFO [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddbd8b36972901bace84c2cfb9622f35 columnFamilyName f1 2024-11-15T00:43:46,937 DEBUG [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:46,939 INFO [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] regionserver.HStore(327): Store=ddbd8b36972901bace84c2cfb9622f35/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:46,939 INFO [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,943 INFO [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddbd8b36972901bace84c2cfb9622f35 columnFamilyName f2 2024-11-15T00:43:46,943 DEBUG [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:46,945 INFO [StoreOpener-ddbd8b36972901bace84c2cfb9622f35-1 {}] regionserver.HStore(327): Store=ddbd8b36972901bace84c2cfb9622f35/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:46,945 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,947 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,948 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,949 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,950 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,951 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T00:43:46,953 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,957 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:46,959 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened ddbd8b36972901bace84c2cfb9622f35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64578251, jitterRate=-0.0377090722322464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T00:43:46,959 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:46,961 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for ddbd8b36972901bace84c2cfb9622f35: Running coprocessor pre-open hook at 1731631426931Writing region info on filesystem at 1731631426931Initializing all the Stores at 1731631426933 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631426933Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631426933Cleaning up temporary data from old regions at 1731631426950 (+17 ms)Running coprocessor post-open hooks at 1731631426959 (+9 ms)Region opened successfully at 1731631426960 (+1 ms) 2024-11-15T00:43:46,963 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35., pid=28, masterSystemTime=1731631426922 2024-11-15T00:43:46,969 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,970 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:46,970 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:46,970 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 36002dcef3f83cc251444774b982f917, NAME => 'testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.', STARTKEY => 'c', ENDKEY => ''} 2024-11-15T00:43:46,971 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:46,971 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:46,971 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:46,971 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:46,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:46,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:46,988 INFO [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:46,992 INFO [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36002dcef3f83cc251444774b982f917 columnFamilyName f1 2024-11-15T00:43:46,992 DEBUG [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:46,994 INFO [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] regionserver.HStore(327): Store=36002dcef3f83cc251444774b982f917/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:46,994 INFO [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:46,998 INFO [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36002dcef3f83cc251444774b982f917 columnFamilyName f2 2024-11-15T00:43:46,998 DEBUG [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:47,002 INFO [StoreOpener-36002dcef3f83cc251444774b982f917-1 {}] regionserver.HStore(327): Store=36002dcef3f83cc251444774b982f917/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:47,003 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=24 2024-11-15T00:43:47,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 in 218 msec 2024-11-15T00:43:47,007 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,008 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,011 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, ASSIGN in 403 msec 2024-11-15T00:43:47,012 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,013 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T00:43:47,020 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,030 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:47,031 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1114): Opened 36002dcef3f83cc251444774b982f917; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63637602, jitterRate=-0.051725834608078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T00:43:47,031 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:47,032 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 36002dcef3f83cc251444774b982f917: Running coprocessor pre-open hook at 1731631426971Writing region info on filesystem at 1731631426971Initializing all the Stores at 1731631426973 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631426974 (+1 ms)Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631426988 (+14 ms)Cleaning up temporary data from old regions at 1731631427012 (+24 ms)Running coprocessor post-open hooks at 1731631427031 (+19 ms)Region opened successfully at 1731631427031 2024-11-15T00:43:47,039 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917., pid=27, masterSystemTime=1731631426922 2024-11-15T00:43:47,047 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:47,047 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:47,047 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:47,047 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => b17ecc70fa519806ead8c7410d465b33, NAME => 'testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.', STARTKEY => 'b', ENDKEY => 'c'} 2024-11-15T00:43:47,048 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:47,051 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:47,051 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,051 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:47,060 INFO [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,065 INFO [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b17ecc70fa519806ead8c7410d465b33 columnFamilyName f1 2024-11-15T00:43:47,065 DEBUG [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:47,066 INFO [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] regionserver.HStore(327): Store=b17ecc70fa519806ead8c7410d465b33/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:47,067 INFO [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,070 INFO [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b17ecc70fa519806ead8c7410d465b33 columnFamilyName f2 2024-11-15T00:43:47,070 DEBUG [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:47,072 INFO [StoreOpener-b17ecc70fa519806ead8c7410d465b33-1 {}] regionserver.HStore(327): Store=b17ecc70fa519806ead8c7410d465b33/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:47,072 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,074 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,075 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,076 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,077 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,078 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T00:43:47,082 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-15T00:43:47,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 in 297 msec 2024-11-15T00:43:47,092 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:47,093 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened b17ecc70fa519806ead8c7410d465b33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67237149, jitterRate=0.0019115954637527466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T00:43:47,093 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:47,094 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for b17ecc70fa519806ead8c7410d465b33: Running coprocessor pre-open hook at 1731631427052Writing region info on filesystem at 1731631427052Initializing all the Stores at 1731631427054 (+2 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631427054Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631427060 (+6 ms)Cleaning up temporary data from old regions at 1731631427077 (+17 ms)Running coprocessor post-open hooks at 1731631427094 (+17 ms)Region opened successfully at 1731631427094 2024-11-15T00:43:47,098 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33., pid=29, masterSystemTime=1731631426922 2024-11-15T00:43:47,104 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:47,104 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:47,104 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:47,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, ASSIGN in 486 msec 2024-11-15T00:43:47,104 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 8735d34935ca3f779d7c7c3264ab101e, NAME => 'testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.', STARTKEY => '', ENDKEY => 'a'} 2024-11-15T00:43:47,105 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,105 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:43:47,106 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,106 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,112 INFO [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:47,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:47,122 INFO [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8735d34935ca3f779d7c7c3264ab101e columnFamilyName f1 2024-11-15T00:43:47,122 DEBUG [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:47,125 INFO [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] regionserver.HStore(327): Store=8735d34935ca3f779d7c7c3264ab101e/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:47,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=25 2024-11-15T00:43:47,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 in 333 msec 2024-11-15T00:43:47,126 INFO [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,129 INFO [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8735d34935ca3f779d7c7c3264ab101e columnFamilyName f2 2024-11-15T00:43:47,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, ASSIGN in 523 msec 2024-11-15T00:43:47,129 DEBUG [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:43:47,131 INFO [StoreOpener-8735d34935ca3f779d7c7c3264ab101e-1 {}] regionserver.HStore(327): Store=8735d34935ca3f779d7c7c3264ab101e/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:43:47,131 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,133 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,134 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,135 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,136 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,137 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-15T00:43:47,140 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,172 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:43:47,177 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 8735d34935ca3f779d7c7c3264ab101e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61322426, jitterRate=-0.08622464537620544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-15T00:43:47,177 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:47,177 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 8735d34935ca3f779d7c7c3264ab101e: Running coprocessor pre-open hook at 1731631427106Writing region info on filesystem at 1731631427106Initializing all the Stores at 1731631427109 (+3 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631427109Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631427112 (+3 ms)Cleaning up temporary data from old regions at 1731631427136 (+24 ms)Running coprocessor post-open hooks at 1731631427177 (+41 ms)Region opened successfully at 1731631427177 2024-11-15T00:43:47,188 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e., pid=30, masterSystemTime=1731631426922 2024-11-15T00:43:47,192 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:47,192 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:47,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:47,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:47,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=23 2024-11-15T00:43:47,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 in 412 msec 2024-11-15T00:43:47,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-15T00:43:47,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, ASSIGN in 617 msec 2024-11-15T00:43:47,246 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:43:47,248 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631427246"}]},"ts":"1731631427246"} 2024-11-15T00:43:47,252 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLED in hbase:meta 2024-11-15T00:43:47,258 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:43:47,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution in 1.6080 sec 2024-11-15T00:43:47,314 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:43:47,409 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-15T00:43:47,413 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T00:43:47,414 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,414 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution' 2024-11-15T00:43:47,421 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:47,421 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:47,422 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:47,665 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,666 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:47,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:47,666 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:47,666 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:47,667 ERROR [Time-limited test {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] 2024-11-15T00:43:47,667 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:47,667 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:47,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:47,668 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:47,669 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:47,674 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[] 2024-11-15T00:43:47,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:47,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:47,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:47,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:47,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:47,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:47,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:47,680 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,681 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:47,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-15T00:43:47,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:47,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:47,685 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:47,685 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:47,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:47,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:47,753 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631457753 2024-11-15T00:43:47,753 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:47,755 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:47,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:47,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:47,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:47,766 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:47,766 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631487766 2024-11-15T00:43:47,766 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631547766 2024-11-15T00:43:47,766 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-15T00:43:47,768 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:47,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:47,783 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:47,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:47,783 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:47,851 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T00:43:47,851 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T00:43:47,854 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:43:47,854 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T00:43:47,855 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T00:43:47,855 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T00:43:47,857 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution 2024-11-15T00:43:47,857 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution Metrics about Tables on a single HBase RegionServer 2024-11-15T00:43:48,017 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,017 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:48,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:48,017 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:48,017 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:48,017 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:48,017 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:48,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:48,018 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:48,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:48,022 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0] 2024-11-15T00:43:48,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:48,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:48,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:48,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:48,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:48,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T00:43:48,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,028 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:48,028 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:48,049 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:48,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:48,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:48,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:48,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:48,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631458054 2024-11-15T00:43:48,054 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:48,055 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,061 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:48,061 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631488061 2024-11-15T00:43:48,061 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631548061 2024-11-15T00:43:48,061 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-15T00:43:48,062 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:48,064 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631428063"}]},"ts":"1731631428063"} 2024-11-15T00:43:48,068 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta 2024-11-15T00:43:48,068 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING 2024-11-15T00:43:48,068 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:48,068 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:48,068 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:48,312 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,312 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:48,312 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:48,312 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:48,312 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:48,312 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:48,312 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:48,312 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:48,313 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:48,315 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:48,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1] 2024-11-15T00:43:48,319 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:48,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:48,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:48,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:48,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,322 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,323 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,323 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:48,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-15T00:43:48,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,325 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:48,326 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:48,363 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:48,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:48,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:48,400 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:48,400 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:48,402 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631458402 2024-11-15T00:43:48,402 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,402 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:48,408 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,410 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:48,410 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631488410 2024-11-15T00:43:48,410 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631548410 2024-11-15T00:43:48,410 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-15T00:43:48,411 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631428410"}]},"ts":"1731631428410"} 2024-11-15T00:43:48,414 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta 2024-11-15T00:43:48,414 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING 2024-11-15T00:43:48,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:48,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}] 2024-11-15T00:43:48,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:48,416 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:48,417 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:48,660 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,660 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:48,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:48,661 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:48,661 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:48,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:48,661 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:48,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:48,661 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:48,662 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:48,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2] 2024-11-15T00:43:48,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:48,666 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:48,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:48,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:48,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:48,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,668 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:48,668 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T00:43:48,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,670 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:48,670 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:48,711 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:48,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:48,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:48,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:48,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:48,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631458749 2024-11-15T00:43:48,749 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:48,749 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,755 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:48,756 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:48,756 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631488756 2024-11-15T00:43:48,756 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631548756 2024-11-15T00:43:48,756 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-15T00:43:48,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}] 2024-11-15T00:43:48,756 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:48,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN}] 2024-11-15T00:43:48,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:48,764 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:48,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:48,764 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:49,006 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,006 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:49,006 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,006 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:49,006 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:49,007 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:49,007 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:49,007 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:49,007 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:49,008 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:49,010 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:49,011 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[] 2024-11-15T00:43:49,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:49,011 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:49,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:49,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:49,012 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,013 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,013 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,014 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,014 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T00:43:49,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,016 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:49,016 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:49,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,043 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631459043 2024-11-15T00:43:49,043 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,043 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:49,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,049 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:49,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631489049 2024-11-15T00:43:49,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631549050 2024-11-15T00:43:49,050 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-15T00:43:49,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN}] 2024-11-15T00:43:49,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:49,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,056 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,057 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,058 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,058 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN because future has completed 2024-11-15T00:43:49,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:49,061 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:49,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:49,061 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:49,061 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,062 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:49,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:49,065 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007faca8bd41d8@6e8df4eb rejected from java.util.concurrent.ThreadPoolExecutor@7aa5b51[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 2] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:43:49,163 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,164 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,268 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,268 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,300 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,300 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:49,300 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,300 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:49,300 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:49,301 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:49,301 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:49,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:49,301 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:49,303 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:49,306 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:49,306 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:49,307 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,307 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,307 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,307 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,307 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:49,308 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:49,309 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:49,309 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:49,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,311 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,312 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,313 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,313 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,313 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,313 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,313 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,313 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,313 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,313 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-15T00:43:49,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,317 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:49,317 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:49,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e to restore RIT 2024-11-15T00:43:49,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:49,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:49,338 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917 to restore RIT 2024-11-15T00:43:49,339 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631459339 2024-11-15T00:43:49,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:49,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,345 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:49,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631489345 2024-11-15T00:43:49,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631549345 2024-11-15T00:43:49,345 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-15T00:43:49,345 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN because future has completed 2024-11-15T00:43:49,351 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:49,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:49,351 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:49,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:49,353 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:49,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:49,353 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:49,354 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,354 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:49,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:49,354 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007faca8bd41d8@4791942e rejected from java.util.concurrent.ThreadPoolExecutor@443b0293[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 2] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:43:49,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN because future has completed 2024-11-15T00:43:49,356 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$840/0x00007faca8bd41d8@d09d570 rejected from java.util.concurrent.ThreadPoolExecutor@443b0293[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 2] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:43:49,371 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,372 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,475 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,476 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,579 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,579 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,595 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,595 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:49,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,595 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:49,596 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:49,596 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:49,596 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:49,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:49,596 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:49,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T00:43:49,600 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:49,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:49,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:49,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:49,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,605 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,608 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,608 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,608 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,608 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-15T00:43:49,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,610 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:49,610 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:49,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,633 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e to restore RIT 2024-11-15T00:43:49,633 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:49,633 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:49,633 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917 to restore RIT 2024-11-15T00:43:49,650 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631459650 2024-11-15T00:43:49,651 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:49,653 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,653 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,653 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,653 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,669 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:49,669 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631489669 2024-11-15T00:43:49,669 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631549669 2024-11-15T00:43:49,670 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-15T00:43:49,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:49,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:49,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN because future has completed 2024-11-15T00:43:49,683 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,683 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,683 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,683 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,693 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:49,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:49,693 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:49,703 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:49,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:49,703 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:49,703 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,703 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:49,787 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,834 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:49,834 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:43:49,835 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 36002dcef3f83cc251444774b982f917, disabling compactions & flushes 2024-11-15T00:43:49,835 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:49,835 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:49,835 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. after waiting 0 ms 2024-11-15T00:43:49,835 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:49,864 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:43:49,867 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917. 2024-11-15T00:43:49,867 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 36002dcef3f83cc251444774b982f917: Waiting for close lock at 1731631429834Running coprocessor pre-close hooks at 1731631429834Disabling compacts and flushes for region at 1731631429834Disabling writes for close at 1731631429835 (+1 ms)Writing region close event to WAL at 1731631429836 (+1 ms)Running coprocessor post-close hooks at 1731631429866 (+30 ms)Closed at 1731631429867 (+1 ms) 2024-11-15T00:43:49,871 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 36002dcef3f83cc251444774b982f917 2024-11-15T00:43:49,871 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(122): Close 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:49,872 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:43:49,872 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1722): Closing 8735d34935ca3f779d7c7c3264ab101e, disabling compactions & flushes 2024-11-15T00:43:49,872 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:49,872 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:49,872 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. after waiting 0 ms 2024-11-15T00:43:49,872 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:49,892 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:49,898 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:43:49,908 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e. 2024-11-15T00:43:49,909 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1676): Region close journal for 8735d34935ca3f779d7c7c3264ab101e: Waiting for close lock at 1731631429872Running coprocessor pre-close hooks at 1731631429872Disabling compacts and flushes for region at 1731631429872Disabling writes for close at 1731631429872Writing region close event to WAL at 1731631429880 (+8 ms)Running coprocessor post-close hooks at 1731631429908 (+28 ms)Closed at 1731631429908 2024-11-15T00:43:49,918 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(157): Closed 8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:49,920 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,920 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:49,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:49,920 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:49,920 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:49,924 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:49,924 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:49,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:49,925 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:49,932 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6] 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:49,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7] 2024-11-15T00:43:49,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8] 2024-11-15T00:43:49,936 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:49,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:49,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:49,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:49,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:49,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:49,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:49,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:49,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:49,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:49,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-15T00:43:49,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:49,945 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:49,946 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:49,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:49,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:49,991 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e to restore RIT 2024-11-15T00:43:49,991 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:49,991 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:49,991 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917 to restore RIT 2024-11-15T00:43:49,996 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631459996 2024-11-15T00:43:49,997 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:49,999 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,000 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,000 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,000 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,014 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,015 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,015 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:50,015 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631490015 2024-11-15T00:43:50,015 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631550015 2024-11-15T00:43:50,016 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-15T00:43:50,023 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED 2024-11-15T00:43:50,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:50,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=CLOSED 2024-11-15T00:43:50,046 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:50,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:50,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=33 2024-11-15T00:43:50,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 in 695 msec 2024-11-15T00:43:50,055 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:50,055 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:50,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,055 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:50,266 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,266 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:50,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,266 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:50,266 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:50,267 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:50,267 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:50,267 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:50,268 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:50,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-15T00:43:50,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7] 2024-11-15T00:43:50,274 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8] 2024-11-15T00:43:50,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:50,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:50,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:50,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:50,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,277 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,277 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,279 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,280 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,280 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,280 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-15T00:43:50,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,282 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:50,282 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:50,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:50,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:50,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:50,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:50,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e to restore RIT 2024-11-15T00:43:50,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:50,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:50,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917 to restore RIT 2024-11-15T00:43:50,307 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631460307 2024-11-15T00:43:50,307 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:50,307 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:50,307 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,307 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,308 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,325 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:50,325 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631490325 2024-11-15T00:43:50,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631550326 2024-11-15T00:43:50,326 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-15T00:43:50,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:50,328 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:50,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN in 1.2780 sec 2024-11-15T00:43:50,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=CLOSED, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:50,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:50,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:50,351 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:50,351 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=38, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,352 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=38, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:50,420 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,420 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,523 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,523 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,576 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 9 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,576 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:50,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,576 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:50,576 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:50,577 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:50,577 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:50,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:50,577 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:50,579 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:50,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7] 2024-11-15T00:43:50,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8] 2024-11-15T00:43:50,583 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:50,583 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:50,583 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:50,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:50,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,587 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,587 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,587 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,588 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-15T00:43:50,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,590 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:50,590 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:50,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:50,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:50,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:50,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:50,611 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:50,611 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:50,611 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917 to restore RIT 2024-11-15T00:43:50,612 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631460612 2024-11-15T00:43:50,612 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:50,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,617 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:50,617 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631490617 2024-11-15T00:43:50,617 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631550617 2024-11-15T00:43:50,617 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-15T00:43:50,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=36002dcef3f83cc251444774b982f917, regionState=CLOSED, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:50,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:50,627 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:50,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:50,628 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,628 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=36 2024-11-15T00:43:50,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 in 934 msec 2024-11-15T00:43:50,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:50,637 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:50,730 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,730 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,833 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,833 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:50,867 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 10 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,867 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:50,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,868 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:50,868 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:50,868 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:50,868 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:50,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:50,868 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:50,869 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:50,871 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:50,871 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:50,872 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:50,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:50,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:50,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:50,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:50,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,877 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:50,877 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:50,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-15T00:43:50,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,879 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:50,879 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:50,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:50,895 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:50,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:50,895 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:50,899 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:50,899 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:50,899 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917 to restore RIT 2024-11-15T00:43:50,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631460900 2024-11-15T00:43:50,900 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:50,900 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,900 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:50,900 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:50,905 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:50,905 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631490905 2024-11-15T00:43:50,905 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631550905 2024-11-15T00:43:50,905 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-15T00:43:50,906 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:50,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:50,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN in 1.8570 sec 2024-11-15T00:43:50,910 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,911 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:50,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:50,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:50,919 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:50,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:50,919 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:50,919 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:50,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:50,919 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:50,935 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:51,038 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:51,141 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:51,156 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 11 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,156 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:51,156 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:51,156 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:51,156 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:51,157 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:51,157 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:51,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:51,157 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:51,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:51,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:51,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:51,161 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:51,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:51,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:51,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:51,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:51,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:51,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:51,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:51,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:51,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:51,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:51,167 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:51,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:51,168 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,168 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T00:43:51,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:51,171 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:51,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:51,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:51,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:51,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:51,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:51,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:51,188 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631461187 2024-11-15T00:43:51,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:51,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,197 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:51,197 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631491197 2024-11-15T00:43:51,197 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631551197 2024-11-15T00:43:51,198 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-15T00:43:51,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,200 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN because future has completed 2024-11-15T00:43:51,207 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:51,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:51,207 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:51,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:51,213 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:51,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:51,213 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:51,213 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:51,213 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:51,362 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(122): Close ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:51,362 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:43:51,362 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1722): Closing ddbd8b36972901bace84c2cfb9622f35, disabling compactions & flushes 2024-11-15T00:43:51,363 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:51,363 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:51,363 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. after waiting 0 ms 2024-11-15T00:43:51,363 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:51,373 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:43:51,374 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35. 2024-11-15T00:43:51,375 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1676): Region close journal for ddbd8b36972901bace84c2cfb9622f35: Waiting for close lock at 1731631431362Running coprocessor pre-close hooks at 1731631431362Disabling compacts and flushes for region at 1731631431362Disabling writes for close at 1731631431363 (+1 ms)Writing region close event to WAL at 1731631431363Running coprocessor post-close hooks at 1731631431374 (+11 ms)Closed at 1731631431374 2024-11-15T00:43:51,378 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(157): Closed ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:51,448 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 12 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:51,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:51,448 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:51,448 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:51,449 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:51,449 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:51,449 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:51,449 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:51,449 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:51,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:51,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:51,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:51,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:51,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:51,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:51,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:51,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:51,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:51,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T00:43:51,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,461 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:51,462 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:51,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:51,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:51,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:51,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:51,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:51,513 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:51,516 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631461516 2024-11-15T00:43:51,516 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:51,518 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=39, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,519 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,524 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:51,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631491525 2024-11-15T00:43:51,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631551525 2024-11-15T00:43:51,525 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-15T00:43:51,528 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED 2024-11-15T00:43:51,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:51,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:51,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:51,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=34 2024-11-15T00:43:51,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 in 329 msec 2024-11-15T00:43:51,543 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:51,543 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:51,543 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:51,554 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:51,656 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:51,759 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fb2f6cb7846a,46669,1731631418265, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:43:51,775 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 13 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,775 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:51,775 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:51,775 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:51,775 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:51,775 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:51,776 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:51,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:51,776 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:51,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:51,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:51,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:51,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:51,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:51,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:51,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:51,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,784 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:51,784 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,785 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:51,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-15T00:43:51,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,786 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:51,786 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:51,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:51,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:51,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:51,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:51,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35 to restore RIT 2024-11-15T00:43:51,841 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:51,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631461842 2024-11-15T00:43:51,842 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:51,842 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,842 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:51,847 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:51,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631491847 2024-11-15T00:43:51,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631551847 2024-11-15T00:43:51,847 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-15T00:43:51,847 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:51,849 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:51,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN in 2.7990 sec 2024-11-15T00:43:51,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:51,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:51,856 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:51,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:51,856 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:51,856 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:51,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:51,856 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:52,097 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 14 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,097 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:52,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:52,097 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:52,097 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:52,098 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:52,098 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:52,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:52,098 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:52,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:52,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:52,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:52,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:52,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:52,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:52,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:52,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T00:43:52,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:52,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:52,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:52,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:52,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:52,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:52,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:52,106 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,107 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T00:43:52,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,108 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:52,108 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:52,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:52,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:52,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:52,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:52,159 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:52,159 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631462159 2024-11-15T00:43:52,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:52,159 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,163 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:52,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631492164 2024-11-15T00:43:52,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631552164 2024-11-15T00:43:52,164 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-15T00:43:52,164 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSING, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN because future has completed 2024-11-15T00:43:52,168 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:43:52,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265}] 2024-11-15T00:43:52,168 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:52,323 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:52,323 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:43:52,323 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing b17ecc70fa519806ead8c7410d465b33, disabling compactions & flushes 2024-11-15T00:43:52,323 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:52,323 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:52,324 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. after waiting 0 ms 2024-11-15T00:43:52,324 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:52,333 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:43:52,334 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33. 2024-11-15T00:43:52,335 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for b17ecc70fa519806ead8c7410d465b33: Waiting for close lock at 1731631432323Running coprocessor pre-close hooks at 1731631432323Disabling compacts and flushes for region at 1731631432323Disabling writes for close at 1731631432324 (+1 ms)Writing region close event to WAL at 1731631432324Running coprocessor post-close hooks at 1731631432334 (+10 ms)Closed at 1731631432334 2024-11-15T00:43:52,338 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:52,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED 2024-11-15T00:43:52,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:52,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:52,343 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=40, ppid=35, state=SUCCESS, hasLock=true; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:52,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=40, ppid=35, state=SUCCESS, hasLock=true; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:52,414 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 15 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,414 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:52,414 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:52,415 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:52,415 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:52,416 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:52,416 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:52,416 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:52,417 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:52,419 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-15T00:43:52,422 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:52,422 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:52,422 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:52,423 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:52,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:52,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:52,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:52,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:52,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:52,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:52,429 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T00:43:52,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,429 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:52,429 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:52,433 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:43:52,476 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:52,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:52,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:52,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:52,480 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:52,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:52,481 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631462481 2024-11-15T00:43:52,481 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:52,481 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,485 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:52,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631492485 2024-11-15T00:43:52,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631552486 2024-11-15T00:43:52,486 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-15T00:43:52,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, regionLocation=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 because future has completed 2024-11-15T00:43:52,490 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:52,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=35 2024-11-15T00:43:52,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 in 322 msec 2024-11-15T00:43:52,493 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:52,493 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:52,494 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:52,736 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 16 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,736 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:52,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:52,736 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:52,736 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:52,736 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:52,737 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:52,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:52,737 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:52,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:52,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:52,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:52,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:52,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:52,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:52,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,741 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:52,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:52,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T00:43:52,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:52,745 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:52,787 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:52,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:52,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:52,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:52,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:52,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33 to restore RIT 2024-11-15T00:43:52,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631462836 2024-11-15T00:43:52,836 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:52,836 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:52,842 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:52,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631492842 2024-11-15T00:43:52,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631552842 2024-11-15T00:43:52,842 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-15T00:43:52,842 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:52,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:52,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=32 2024-11-15T00:43:52,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN in 3.7940 sec 2024-11-15T00:43:52,848 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:52,848 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:52,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:52,848 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:53,093 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 17 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,093 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:53,093 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,093 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:53,093 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:53,093 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:53,093 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:53,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:53,094 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:53,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:53,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:53,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:53,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:53,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:53,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:53,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:53,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:53,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,102 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T00:43:53,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,103 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:53,103 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:53,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:53,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:53,125 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:53,125 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:53,137 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,137 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631463137 2024-11-15T00:43:53,137 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:53,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,142 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,142 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:53,143 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631493142 2024-11-15T00:43:53,143 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631553143 2024-11-15T00:43:53,143 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-15T00:43:53,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:53,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T00:43:53,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution in 4.3870 sec 2024-11-15T00:43:53,149 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:53,149 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,149 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:53,393 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 18 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,393 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:53,393 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,393 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:53,393 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:53,393 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:53,394 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:53,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:53,394 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:53,395 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:53,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:53,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:53,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:53,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:53,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,399 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T00:43:53,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,400 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:53,400 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:53,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:53,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:53,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:53,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:53,411 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631463411 2024-11-15T00:43:53,411 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,411 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:53,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,415 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:53,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631493415 2024-11-15T00:43:53,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631553415 2024-11-15T00:43:53,416 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-15T00:43:53,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:53,418 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631433417"}]},"ts":"1731631433417"} 2024-11-15T00:43:53,420 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta 2024-11-15T00:43:53,420 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED 2024-11-15T00:43:53,420 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:53,420 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,420 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:53,666 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 19 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,666 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:53,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,666 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:53,666 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:53,667 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:53,667 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:53,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:53,667 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:53,668 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:43:53,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22] 2024-11-15T00:43:53,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-15T00:43:53,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:53,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:53,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:53,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:53,671 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-15T00:43:53,671 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-15T00:43:53,671 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:53,671 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:53,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:53,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:53,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,676 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T00:43:53,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,677 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:53,677 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:53,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:53,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:53,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:53,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:53,706 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631463706 2024-11-15T00:43:53,706 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:53,706 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,711 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:53,711 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631493711 2024-11-15T00:43:53,711 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631553711 2024-11-15T00:43:53,711 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-15T00:43:53,712 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631433711"}]},"ts":"1731631433711"} 2024-11-15T00:43:53,715 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta 2024-11-15T00:43:53,715 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED 2024-11-15T00:43:53,715 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:53,717 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-15T00:43:53,717 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,717 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:53,961 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 20 exec state=pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,961 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T00:43:53,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:53,962 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:53,962 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:53,962 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T00:43:53,962 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T00:43:53,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T00:43:53,962 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:43:53,963 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T00:43:53,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22, 23] 2024-11-15T00:43:53,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 10] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 9] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[7, 12] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[8, 11] 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-15T00:43:53,965 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-15T00:43:53,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-15T00:43:53,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-15T00:43:53,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=8735d34935ca3f779d7c7c3264ab101e, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=ddbd8b36972901bace84c2cfb9622f35, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b17ecc70fa519806ead8c7410d465b33, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=36002dcef3f83cc251444774b982f917, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 8735d34935ca3f779d7c7c3264ab101e, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 36002dcef3f83cc251444774b982f917, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure ddbd8b36972901bace84c2cfb9622f35, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure b17ecc70fa519806ead8c7410d465b33, server=fb2f6cb7846a,46669,1731631418265 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T00:43:53,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T00:43:53,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:53,969 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fb2f6cb7846a,46669,1731631418265, table=hbase:meta, region=1588230740 2024-11-15T00:43:53,969 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,46669,1731631418265, state=OPEN 2024-11-15T00:43:54,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:54,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:43:54,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:54,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:43:54,033 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631464033 2024-11-15T00:43:54,033 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:43:54,033 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8735d34935ca3f779d7c7c3264ab101e, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:54,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddbd8b36972901bace84c2cfb9622f35, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:54,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b17ecc70fa519806ead8c7410d465b33, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:54,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36002dcef3f83cc251444774b982f917, regionState=CLOSED, lastHost=fb2f6cb7846a,46669,1731631418265, regionLocation=fb2f6cb7846a,46669,1731631418265, openSeqNum=2 2024-11-15T00:43:54,036 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:43:54,036 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631494036 2024-11-15T00:43:54,036 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631554036 2024-11-15T00:43:54,036 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-15T00:43:54,036 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-15T00:43:54,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution in 6.6270 sec 2024-11-15T00:43:54,287 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-15T00:43:54,288 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable testRecoveryAndDoubleExecution 2024-11-15T00:43:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:43:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] ipc.CallRunner(138): callId: 13 service: MasterService methodName: DisableTable size: 92 connection: 172.17.0.3:52328 deadline: 1731631494289, exception=org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution 2024-11-15T00:43:54,291 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testRecoveryAndDoubleExecution failed with org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:43:54,291 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testRecoveryAndDoubleExecution already disabled, so just deleting it. 2024-11-15T00:43:54,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete testRecoveryAndDoubleExecution 2024-11-15T00:43:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,295 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:43:54,296 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,306 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:54,306 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:54,307 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:54,310 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/f1, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/f2, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/recovered.edits] 2024-11-15T00:43:54,310 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/f1, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/f2, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/recovered.edits] 2024-11-15T00:43:54,310 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/f1, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/f2, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/recovered.edits] 2024-11-15T00:43:54,311 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917 2024-11-15T00:43:54,315 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/f1, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/f2, FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/recovered.edits] 2024-11-15T00:43:54,321 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/recovered.edits/4.seqid to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/archive/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e/recovered.edits/4.seqid 2024-11-15T00:43:54,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/8735d34935ca3f779d7c7c3264ab101e 2024-11-15T00:43:54,323 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/recovered.edits/4.seqid to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/archive/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33/recovered.edits/4.seqid 2024-11-15T00:43:54,324 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/b17ecc70fa519806ead8c7410d465b33 2024-11-15T00:43:54,325 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/recovered.edits/4.seqid to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/archive/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35/recovered.edits/4.seqid 2024-11-15T00:43:54,326 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/ddbd8b36972901bace84c2cfb9622f35 2024-11-15T00:43:54,327 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/recovered.edits/4.seqid to hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/archive/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917/recovered.edits/4.seqid 2024-11-15T00:43:54,327 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/default/testRecoveryAndDoubleExecution/36002dcef3f83cc251444774b982f917 2024-11-15T00:43:54,327 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testRecoveryAndDoubleExecution regions 2024-11-15T00:43:54,331 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,336 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 4 rows of testRecoveryAndDoubleExecution from hbase:meta 2024-11-15T00:43:54,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testRecoveryAndDoubleExecution' descriptor. 2024-11-15T00:43:54,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testRecoveryAndDoubleExecution' from region states. 2024-11-15T00:43:54,344 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631434344"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:54,344 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631434344"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:54,344 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631434344"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:54,345 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631434344"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:54,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 4 regions from META 2024-11-15T00:43:54,349 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8735d34935ca3f779d7c7c3264ab101e, NAME => 'testRecoveryAndDoubleExecution,,1731631425650.8735d34935ca3f779d7c7c3264ab101e.', STARTKEY => '', ENDKEY => 'a'}, {ENCODED => ddbd8b36972901bace84c2cfb9622f35, NAME => 'testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35.', STARTKEY => 'a', ENDKEY => 'b'}, {ENCODED => b17ecc70fa519806ead8c7410d465b33, NAME => 'testRecoveryAndDoubleExecution,b,1731631425650.b17ecc70fa519806ead8c7410d465b33.', STARTKEY => 'b', ENDKEY => 'c'}, {ENCODED => 36002dcef3f83cc251444774b982f917, NAME => 'testRecoveryAndDoubleExecution,c,1731631425650.36002dcef3f83cc251444774b982f917.', STARTKEY => 'c', ENDKEY => ''}] 2024-11-15T00:43:54,349 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testRecoveryAndDoubleExecution' as deleted. 2024-11-15T00:43:54,350 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631434349"}]},"ts":"9223372036854775807"} 2024-11-15T00:43:54,353 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testRecoveryAndDoubleExecution state from META 2024-11-15T00:43:54,354 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-15T00:43:54,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution in 62 msec 2024-11-15T00:43:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:43:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testRecoveryAndDoubleExecution 2024-11-15T00:43:54,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testRecoveryAndDoubleExecution completed 2024-11-15T00:43:54,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=217 (was 220), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 453) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1977 (was 3262) 2024-11-15T00:43:54,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T00:43:54,412 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:43:54,413 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:43:54,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:43:54,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:43:54,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:43:54,419 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T00:43:54,419 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1880379111, stopped=false 2024-11-15T00:43:54,419 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fb2f6cb7846a,41375,1731631417476 2024-11-15T00:43:54,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T00:43:54,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T00:43:54,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:54,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:54,442 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:54,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:43:54,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:43:54,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:43:54,442 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:43:54,442 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:43:54,443 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fb2f6cb7846a,46669,1731631418265' ***** 2024-11-15T00:43:54,443 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T00:43:54,443 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T00:43:54,444 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(959): stopping server fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fb2f6cb7846a:46669. 2024-11-15T00:43:54,444 DEBUG [RS:0;fb2f6cb7846a:46669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:43:54,444 DEBUG [RS:0;fb2f6cb7846a:46669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T00:43:54,444 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T00:43:54,445 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T00:43:54,445 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T00:43:54,445 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:54,445 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T00:43:54,445 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T00:43:54,445 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T00:43:54,445 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T00:43:54,445 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T00:43:54,448 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=24.62 KB heapSize=40.34 KB 2024-11-15T00:43:54,503 INFO [regionserver/fb2f6cb7846a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T00:43:54,517 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/info/8469037348734c7fab1439f1037c0160 is 108, key is testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35./info:/1731631434344/DeleteFamily/seqid=0 2024-11-15T00:43:54,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741844_1020 (size=6669) 2024-11-15T00:43:54,646 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:54,846 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:54,875 INFO [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T00:43:54,875 INFO [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T00:43:54,931 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.79 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/info/8469037348734c7fab1439f1037c0160 2024-11-15T00:43:54,992 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/ns/eebd646ed82b4f6cb0abcbf6d350097e is 106, key is testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35./ns:/1731631434331/DeleteFamily/seqid=0 2024-11-15T00:43:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741845_1021 (size=6009) 2024-11-15T00:43:55,046 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:55,246 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:55,400 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/ns/eebd646ed82b4f6cb0abcbf6d350097e 2024-11-15T00:43:55,429 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/rep_barrier/ef8c61d88b504769bb31e6fab9617479 is 115, key is testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35./rep_barrier:/1731631434331/DeleteFamily/seqid=0 2024-11-15T00:43:55,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741846_1022 (size=6121) 2024-11-15T00:43:55,447 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:55,647 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:55,835 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=648 B at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/rep_barrier/ef8c61d88b504769bb31e6fab9617479 2024-11-15T00:43:55,847 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:55,861 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/table/16df54efd20f4f9ca113654a23fea492 is 109, key is testRecoveryAndDoubleExecution,a,1731631425650.ddbd8b36972901bace84c2cfb9622f35./table:/1731631434331/DeleteFamily/seqid=0 2024-11-15T00:43:55,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741847_1023 (size=6158) 2024-11-15T00:43:56,048 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:56,248 DEBUG [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:43:56,267 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.55 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/table/16df54efd20f4f9ca113654a23fea492 2024-11-15T00:43:56,278 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/info/8469037348734c7fab1439f1037c0160 as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/info/8469037348734c7fab1439f1037c0160 2024-11-15T00:43:56,288 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/info/8469037348734c7fab1439f1037c0160, entries=12, sequenceid=76, filesize=6.5 K 2024-11-15T00:43:56,291 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/ns/eebd646ed82b4f6cb0abcbf6d350097e as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/ns/eebd646ed82b4f6cb0abcbf6d350097e 2024-11-15T00:43:56,298 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/ns/eebd646ed82b4f6cb0abcbf6d350097e, entries=8, sequenceid=76, filesize=5.9 K 2024-11-15T00:43:56,300 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/rep_barrier/ef8c61d88b504769bb31e6fab9617479 as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/rep_barrier/ef8c61d88b504769bb31e6fab9617479 2024-11-15T00:43:56,307 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/rep_barrier/ef8c61d88b504769bb31e6fab9617479, entries=6, sequenceid=76, filesize=6.0 K 2024-11-15T00:43:56,308 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/.tmp/table/16df54efd20f4f9ca113654a23fea492 as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/table/16df54efd20f4f9ca113654a23fea492 2024-11-15T00:43:56,316 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/table/16df54efd20f4f9ca113654a23fea492, entries=9, sequenceid=76, filesize=6.0 K 2024-11-15T00:43:56,324 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~24.62 KB/25215, heapSize ~40.28 KB/41248, currentSize=0 B/0 for 1588230740 in 1871ms, sequenceid=76, compaction requested=false 2024-11-15T00:43:56,332 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/data/hbase/meta/1588230740/recovered.edits/79.seqid, newMaxSeqId=79, maxSeqId=1 2024-11-15T00:43:56,333 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:43:56,333 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T00:43:56,333 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731631434445Running coprocessor pre-close hooks at 1731631434445Disabling compacts and flushes for region at 1731631434445Disabling writes for close at 1731631434445Obtaining lock to block concurrent updates at 1731631434449 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1731631434449Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=25215, getHeapSize=41248, getOffHeapSize=0, getCellsCount=199 at 1731631434455 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731631434457 (+2 ms)Flushing 1588230740/info: creating writer at 1731631434460 (+3 ms)Flushing 1588230740/info: appending metadata at 1731631434504 (+44 ms)Flushing 1588230740/info: closing flushed file at 1731631434506 (+2 ms)Flushing 1588230740/ns: creating writer at 1731631434975 (+469 ms)Flushing 1588230740/ns: appending metadata at 1731631434991 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731631434991Flushing 1588230740/rep_barrier: creating writer at 1731631435412 (+421 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731631435429 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731631435429Flushing 1588230740/table: creating writer at 1731631435844 (+415 ms)Flushing 1588230740/table: appending metadata at 1731631435860 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731631435860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@478e5ca4: reopening flushed file at 1731631436276 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@791222ce: reopening flushed file at 1731631436290 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70042332: reopening flushed file at 1731631436299 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e2062f5: reopening flushed file at 1731631436307 (+8 ms)Finished flush of dataSize ~24.62 KB/25215, heapSize ~40.28 KB/41248, currentSize=0 B/0 for 1588230740 in 1871ms, sequenceid=76, compaction requested=false at 1731631436324 (+17 ms)Writing region close event to WAL at 1731631436327 (+3 ms)Running coprocessor post-close hooks at 1731631436333 (+6 ms)Closed at 1731631436333 2024-11-15T00:43:56,334 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T00:43:56,448 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(976): stopping server fb2f6cb7846a,46669,1731631418265; all regions closed. 2024-11-15T00:43:56,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741834_1010 (size=29703) 2024-11-15T00:43:56,458 DEBUG [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/oldWALs 2024-11-15T00:43:56,458 INFO [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fb2f6cb7846a%2C46669%2C1731631418265.meta:.meta(num 1731631421749) 2024-11-15T00:43:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741833_1009 (size=4131) 2024-11-15T00:43:56,461 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/WALs/fb2f6cb7846a,46669,1731631418265/fb2f6cb7846a%2C46669%2C1731631418265.1731631421055 not finished, retry = 0 2024-11-15T00:43:56,564 DEBUG [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/oldWALs 2024-11-15T00:43:56,565 INFO [RS:0;fb2f6cb7846a:46669 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fb2f6cb7846a%2C46669%2C1731631418265:(num 1731631421055) 2024-11-15T00:43:56,565 DEBUG [RS:0;fb2f6cb7846a:46669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:43:56,565 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T00:43:56,565 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T00:43:56,565 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.ChoreService(370): Chore service for: regionserver/fb2f6cb7846a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T00:43:56,565 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T00:43:56,565 INFO [regionserver/fb2f6cb7846a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T00:43:56,566 INFO [RS:0;fb2f6cb7846a:46669 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46669 2024-11-15T00:43:56,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fb2f6cb7846a,46669,1731631418265 2024-11-15T00:43:56,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T00:43:56,620 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T00:43:56,632 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fb2f6cb7846a,46669,1731631418265] 2024-11-15T00:43:56,642 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fb2f6cb7846a,46669,1731631418265 already deleted, retry=false 2024-11-15T00:43:56,642 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fb2f6cb7846a,46669,1731631418265 expired; onlineServers=0 2024-11-15T00:43:56,642 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fb2f6cb7846a,41375,1731631417476' ***** 2024-11-15T00:43:56,642 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T00:43:56,643 INFO [M:0;fb2f6cb7846a:41375 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T00:43:56,643 INFO [M:0;fb2f6cb7846a:41375 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T00:43:56,643 DEBUG [M:0;fb2f6cb7846a:41375 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T00:43:56,644 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T00:43:56,644 DEBUG [M:0;fb2f6cb7846a:41375 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T00:43:56,644 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631420671 {}] cleaner.HFileCleaner(306): Exit Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631420671,5,FailOnTimeoutGroup] 2024-11-15T00:43:56,644 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631420673 {}] cleaner.HFileCleaner(306): Exit Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631420673,5,FailOnTimeoutGroup] 2024-11-15T00:43:56,644 INFO [M:0;fb2f6cb7846a:41375 {}] hbase.ChoreService(370): Chore service for: master/fb2f6cb7846a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T00:43:56,645 INFO [M:0;fb2f6cb7846a:41375 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T00:43:56,645 DEBUG [M:0;fb2f6cb7846a:41375 {}] master.HMaster(1795): Stopping service threads 2024-11-15T00:43:56,645 INFO [M:0;fb2f6cb7846a:41375 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T00:43:56,645 INFO [M:0;fb2f6cb7846a:41375 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:43:56,646 ERROR [M:0;fb2f6cb7846a:41375 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-3,5,PEWorkerGroup] Thread[HFileArchiver-4,5,PEWorkerGroup] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T00:43:56,646 INFO [M:0;fb2f6cb7846a:41375 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:43:56,646 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T00:43:56,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T00:43:56,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:43:56,652 DEBUG [M:0;fb2f6cb7846a:41375 {}] zookeeper.ZKUtil(347): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T00:43:56,653 WARN [M:0;fb2f6cb7846a:41375 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T00:43:56,654 INFO [M:0;fb2f6cb7846a:41375 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/.lastflushedseqids 2024-11-15T00:43:56,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741848_1024 (size=72) 2024-11-15T00:43:56,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:43:56,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46669-0x1013bf2805d0001, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:43:56,732 INFO [RS:0;fb2f6cb7846a:46669 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T00:43:56,733 INFO [RS:0;fb2f6cb7846a:46669 {}] regionserver.HRegionServer(1031): Exiting; stopping=fb2f6cb7846a,46669,1731631418265; zookeeper connection closed. 2024-11-15T00:43:56,733 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5eca9919 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5eca9919 2024-11-15T00:43:56,733 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T00:43:57,068 INFO [M:0;fb2f6cb7846a:41375 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:43:57,069 INFO [M:0;fb2f6cb7846a:41375 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T00:43:57,069 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T00:43:57,069 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:57,069 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:57,069 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T00:43:57,069 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:57,069 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=150.57 KB heapSize=180.66 KB 2024-11-15T00:43:57,087 DEBUG [M:0;fb2f6cb7846a:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0489587536a4aa4bc82ab3b4abf5013 is 82, key is hbase:meta,,1/info:regioninfo/1731631421850/Put/seqid=0 2024-11-15T00:43:57,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741849_1025 (size=5672) 2024-11-15T00:43:57,093 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0489587536a4aa4bc82ab3b4abf5013 2024-11-15T00:43:57,127 DEBUG [M:0;fb2f6cb7846a:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28cdab4015934f599d117e7fc9dcdd9e is 1206, key is \x00\x00\x00\x00\x00\x00\x00\x16/proc:d/1731631427267/Put/seqid=0 2024-11-15T00:43:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741850_1026 (size=11516) 2024-11-15T00:43:57,141 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150.01 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28cdab4015934f599d117e7fc9dcdd9e 2024-11-15T00:43:57,176 DEBUG [M:0;fb2f6cb7846a:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d95fda31403c4afca6366a6d53e458e7 is 69, key is fb2f6cb7846a,46669,1731631418265/rs:state/1731631420719/Put/seqid=0 2024-11-15T00:43:57,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741851_1027 (size=5156) 2024-11-15T00:43:57,585 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d95fda31403c4afca6366a6d53e458e7 2024-11-15T00:43:57,596 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0489587536a4aa4bc82ab3b4abf5013 as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0489587536a4aa4bc82ab3b4abf5013 2024-11-15T00:43:57,606 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0489587536a4aa4bc82ab3b4abf5013, entries=8, sequenceid=332, filesize=5.5 K 2024-11-15T00:43:57,608 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28cdab4015934f599d117e7fc9dcdd9e as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/28cdab4015934f599d117e7fc9dcdd9e 2024-11-15T00:43:57,619 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/28cdab4015934f599d117e7fc9dcdd9e, entries=39, sequenceid=332, filesize=11.2 K 2024-11-15T00:43:57,621 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d95fda31403c4afca6366a6d53e458e7 as hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d95fda31403c4afca6366a6d53e458e7 2024-11-15T00:43:57,630 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34361/user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d95fda31403c4afca6366a6d53e458e7, entries=1, sequenceid=332, filesize=5.0 K 2024-11-15T00:43:57,631 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(3140): Finished flush of dataSize ~150.57 KB/154182, heapSize ~180.36 KB/184688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 562ms, sequenceid=332, compaction requested=false 2024-11-15T00:43:57,633 INFO [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:43:57,633 DEBUG [M:0;fb2f6cb7846a:41375 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731631437069Disabling compacts and flushes for region at 1731631437069Disabling writes for close at 1731631437069Obtaining lock to block concurrent updates at 1731631437069Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731631437069Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=154182, getHeapSize=184928, getOffHeapSize=0, getCellsCount=393 at 1731631437070 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731631437070Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731631437070Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731631437087 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731631437087Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731631437101 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731631437127 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731631437127Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731631437153 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731631437176 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731631437176Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dafe55d: reopening flushed file at 1731631437594 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a8d7701: reopening flushed file at 1731631437606 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1381cfc3: reopening flushed file at 1731631437620 (+14 ms)Finished flush of dataSize ~150.57 KB/154182, heapSize ~180.36 KB/184688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 562ms, sequenceid=332, compaction requested=false at 1731631437631 (+11 ms)Writing region close event to WAL at 1731631437633 (+2 ms)Closed at 1731631437633 2024-11-15T00:43:57,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44695 is added to blk_1073741830_1006 (size=176425) 2024-11-15T00:43:57,637 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/80eb9562-6f9f-630b-1058-c41302b3d4e6/MasterData/WALs/fb2f6cb7846a,41375,1731631417476/fb2f6cb7846a%2C41375%2C1731631417476.1731631419762 not finished, retry = 0 2024-11-15T00:43:57,738 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T00:43:57,738 INFO [M:0;fb2f6cb7846a:41375 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T00:43:57,738 INFO [M:0;fb2f6cb7846a:41375 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41375 2024-11-15T00:43:57,739 INFO [M:0;fb2f6cb7846a:41375 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T00:43:57,838 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:43:57,871 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:43:57,872 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T00:43:57,872 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T00:43:57,872 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution 2024-11-15T00:43:57,888 INFO [M:0;fb2f6cb7846a:41375 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T00:43:57,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:43:57,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1013bf2805d0000, quorum=127.0.0.1:63272, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:43:57,892 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@156ffeaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T00:43:57,894 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T00:43:57,894 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T00:43:57,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T00:43:57,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/hadoop.log.dir/,STOPPED} 2024-11-15T00:43:57,897 WARN [BP-147362170-172.17.0.3-1731631413147 heartbeating to localhost/127.0.0.1:34361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T00:43:57,897 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T00:43:57,897 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T00:43:57,897 WARN [BP-147362170-172.17.0.3-1731631413147 heartbeating to localhost/127.0.0.1:34361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-147362170-172.17.0.3-1731631413147 (Datanode Uuid 2e9bf9a5-71c5-4405-bc8a-d74a59c9044b) service to localhost/127.0.0.1:34361 2024-11-15T00:43:57,898 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/data/data1/current/BP-147362170-172.17.0.3-1731631413147 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T00:43:57,898 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/cluster_f4767149-9e5d-6845-1c9c-620e3676c23f/data/data2/current/BP-147362170-172.17.0.3-1731631413147 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T00:43:57,899 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T00:43:57,907 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27b19a13{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T00:43:57,908 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T00:43:57,908 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T00:43:57,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T00:43:57,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f530453-3803-b305-a6d5-b428040e3564/hadoop.log.dir/,STOPPED} 2024-11-15T00:43:57,916 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T00:43:57,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down