2024-11-22 23:43:35,794 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 2024-11-22 23:43:35,811 main DEBUG Took 0.013866 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 23:43:35,811 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 23:43:35,812 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 23:43:35,817 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 23:43:35,820 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,838 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 23:43:35,872 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,875 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,877 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,878 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,879 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,881 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,881 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,882 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,882 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,883 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,883 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,884 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,884 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,885 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,885 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,886 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,886 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,887 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,887 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:43:35,888 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,888 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 23:43:35,890 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:43:35,892 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 23:43:35,894 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 23:43:35,895 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 23:43:35,896 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 23:43:35,897 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 23:43:35,909 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 23:43:35,913 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 23:43:35,915 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 23:43:35,916 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 23:43:35,916 main DEBUG createAppenders(={Console}) 2024-11-22 23:43:35,917 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 initialized 2024-11-22 23:43:35,918 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 2024-11-22 23:43:35,919 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1c742ed4 OK. 2024-11-22 23:43:35,920 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 23:43:35,920 main DEBUG OutputStream closed 2024-11-22 23:43:35,921 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 23:43:35,921 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 23:43:35,922 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4803b726 OK 2024-11-22 23:43:36,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 23:43:36,026 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 23:43:36,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 23:43:36,028 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 23:43:36,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 23:43:36,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 23:43:36,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 23:43:36,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 23:43:36,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 23:43:36,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 23:43:36,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 23:43:36,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 23:43:36,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 23:43:36,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 23:43:36,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 23:43:36,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 23:43:36,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 23:43:36,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 23:43:36,039 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 23:43:36,039 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null 2024-11-22 23:43:36,040 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 23:43:36,041 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK. 2024-11-22T23:43:36,307 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485 2024-11-22 23:43:36,310 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 23:43:36,310 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T23:43:36,319 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure timeout: 13 mins 2024-11-22T23:43:36,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T23:43:36,352 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76, deleteOnExit=true 2024-11-22T23:43:36,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T23:43:36,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/test.cache.data in system properties and HBase conf 2024-11-22T23:43:36,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T23:43:36,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/hadoop.log.dir in system properties and HBase conf 2024-11-22T23:43:36,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T23:43:36,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T23:43:36,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T23:43:36,441 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T23:43:36,534 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:43:36,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T23:43:36,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T23:43:36,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T23:43:36,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T23:43:36,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T23:43:36,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T23:43:36,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T23:43:36,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T23:43:36,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T23:43:36,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/nfs.dump.dir in system properties and HBase conf 2024-11-22T23:43:36,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/java.io.tmpdir in system properties and HBase conf 2024-11-22T23:43:36,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T23:43:36,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T23:43:36,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T23:43:37,355 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T23:43:37,435 INFO [Time-limited test {}] log.Log(170): Logging initialized @2509ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T23:43:37,512 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:43:37,571 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T23:43:37,591 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T23:43:37,592 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T23:43:37,593 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T23:43:37,605 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:43:37,607 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/hadoop.log.dir/,AVAILABLE} 2024-11-22T23:43:37,608 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T23:43:37,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27b19a13{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/java.io.tmpdir/jetty-localhost-42345-hadoop-hdfs-3_4_1-tests_jar-_-any-12590955615266741557/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T23:43:37,822 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:42345} 2024-11-22T23:43:37,823 INFO [Time-limited test {}] server.Server(415): Started @2897ms 2024-11-22T23:43:38,201 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:43:38,207 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T23:43:38,209 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T23:43:38,209 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T23:43:38,209 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T23:43:38,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/hadoop.log.dir/,AVAILABLE} 2024-11-22T23:43:38,211 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T23:43:38,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@156ffeaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/java.io.tmpdir/jetty-localhost-46481-hadoop-hdfs-3_4_1-tests_jar-_-any-14612558982061365861/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T23:43:38,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:46481} 2024-11-22T23:43:38,330 INFO [Time-limited test {}] server.Server(415): Started @3405ms 2024-11-22T23:43:38,389 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T23:43:38,876 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/data/data2/current/BP-363626384-172.17.0.2-1732319017109/current, will proceed with Du for space computation calculation, 2024-11-22T23:43:38,876 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/data/data1/current/BP-363626384-172.17.0.2-1732319017109/current, will proceed with Du for space computation calculation, 2024-11-22T23:43:38,911 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T23:43:38,962 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3320c249f69d3de2 with lease ID 0x1020bc2ae39ff4aa: Processing first storage report for DS-ffab1390-4579-4e6e-a937-5f1ca90e7ead from datanode DatanodeRegistration(127.0.0.1:45113, datanodeUuid=0efe8e3a-43f8-4371-8c09-d1207ffbc6b7, infoPort=37403, infoSecurePort=0, ipcPort=36735, storageInfo=lv=-57;cid=testClusterID;nsid=208202497;c=1732319017109) 2024-11-22T23:43:38,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3320c249f69d3de2 with lease ID 0x1020bc2ae39ff4aa: from storage DS-ffab1390-4579-4e6e-a937-5f1ca90e7ead node DatanodeRegistration(127.0.0.1:45113, datanodeUuid=0efe8e3a-43f8-4371-8c09-d1207ffbc6b7, infoPort=37403, infoSecurePort=0, ipcPort=36735, storageInfo=lv=-57;cid=testClusterID;nsid=208202497;c=1732319017109), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T23:43:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3320c249f69d3de2 with lease ID 0x1020bc2ae39ff4aa: Processing first storage report for DS-0078f51d-cc3b-4a45-90d6-5c1c31049754 from datanode DatanodeRegistration(127.0.0.1:45113, datanodeUuid=0efe8e3a-43f8-4371-8c09-d1207ffbc6b7, infoPort=37403, infoSecurePort=0, ipcPort=36735, storageInfo=lv=-57;cid=testClusterID;nsid=208202497;c=1732319017109) 2024-11-22T23:43:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3320c249f69d3de2 with lease ID 0x1020bc2ae39ff4aa: from storage DS-0078f51d-cc3b-4a45-90d6-5c1c31049754 node DatanodeRegistration(127.0.0.1:45113, datanodeUuid=0efe8e3a-43f8-4371-8c09-d1207ffbc6b7, infoPort=37403, infoSecurePort=0, ipcPort=36735, storageInfo=lv=-57;cid=testClusterID;nsid=208202497;c=1732319017109), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T23:43:39,049 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485 2024-11-22T23:43:39,123 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/zookeeper_0, clientPort=55056, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T23:43:39,134 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55056 2024-11-22T23:43:39,143 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:39,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741825_1001 (size=7) 2024-11-22T23:43:39,799 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc with version=8 2024-11-22T23:43:39,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/hbase-staging 2024-11-22T23:43:39,886 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T23:43:40,133 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:43:40,144 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:43:40,145 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:43:40,149 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:43:40,149 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:43:40,150 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:43:40,289 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T23:43:40,349 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T23:43:40,361 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T23:43:40,366 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:43:40,395 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9168 (auto-detected) 2024-11-22T23:43:40,396 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T23:43:40,414 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41373 2024-11-22T23:43:40,435 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41373 connecting to ZooKeeper ensemble=127.0.0.1:55056 2024-11-22T23:43:40,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413730x0, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:43:40,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41373-0x10142106af00000 connected 2024-11-22T23:43:40,500 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:40,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:40,511 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:43:40,515 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc, hbase.cluster.distributed=false 2024-11-22T23:43:40,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:43:40,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41373 2024-11-22T23:43:40,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41373 2024-11-22T23:43:40,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41373 2024-11-22T23:43:40,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41373 2024-11-22T23:43:40,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41373 2024-11-22T23:43:40,670 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:43:40,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:43:40,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:43:40,672 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:43:40,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:43:40,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:43:40,676 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:43:40,678 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:43:40,679 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43005 2024-11-22T23:43:40,680 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43005 connecting to ZooKeeper ensemble=127.0.0.1:55056 2024-11-22T23:43:40,682 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:40,684 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:40,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430050x0, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:43:40,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43005-0x10142106af00001 connected 2024-11-22T23:43:40,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:43:40,696 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:43:40,703 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:43:40,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:43:40,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:43:40,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43005 2024-11-22T23:43:40,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43005 2024-11-22T23:43:40,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43005 2024-11-22T23:43:40,713 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43005 2024-11-22T23:43:40,713 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43005 2024-11-22T23:43:40,733 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a511e45b3024:41373 2024-11-22T23:43:40,734 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a511e45b3024,41373,1732319019939 2024-11-22T23:43:40,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:43:40,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:43:40,743 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a511e45b3024,41373,1732319019939 2024-11-22T23:43:40,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:43:40,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:40,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:40,766 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:43:40,767 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a511e45b3024,41373,1732319019939 from backup master directory 2024-11-22T23:43:40,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a511e45b3024,41373,1732319019939 2024-11-22T23:43:40,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:43:40,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:43:40,771 WARN [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:43:40,771 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a511e45b3024,41373,1732319019939 2024-11-22T23:43:40,774 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T23:43:40,776 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T23:43:40,832 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/hbase.id] with ID: e9b0a6ea-23d3-4fcd-89f5-5129f7df5b64 2024-11-22T23:43:40,832 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/.tmp/hbase.id 2024-11-22T23:43:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741826_1002 (size=42) 2024-11-22T23:43:41,245 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/.tmp/hbase.id]:[hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/hbase.id] 2024-11-22T23:43:41,288 INFO [master/a511e45b3024:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:43:41,293 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T23:43:41,311 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-22T23:43:41,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:41,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:41,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741827_1003 (size=196) 2024-11-22T23:43:41,747 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:43:41,748 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T23:43:41,764 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:41,769 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:43:41,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741828_1004 (size=1189) 2024-11-22T23:43:41,816 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store 2024-11-22T23:43:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741829_1005 (size=34) 2024-11-22T23:43:41,839 INFO [master/a511e45b3024:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T23:43:41,842 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:41,843 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:43:41,843 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:41,843 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:41,844 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:43:41,845 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:41,845 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:41,846 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319021843Disabling compacts and flushes for region at 1732319021843Disabling writes for close at 1732319021845 (+2 ms)Writing region close event to WAL at 1732319021845Closed at 1732319021845 2024-11-22T23:43:41,848 WARN [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/.initializing 2024-11-22T23:43:41,848 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/WALs/a511e45b3024,41373,1732319019939 2024-11-22T23:43:41,856 INFO [master/a511e45b3024:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:43:41,869 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C41373%2C1732319019939, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/WALs/a511e45b3024,41373,1732319019939, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/oldWALs, maxLogs=10 2024-11-22T23:43:41,892 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/WALs/a511e45b3024,41373,1732319019939/a511e45b3024%2C41373%2C1732319019939.1732319021874, exclude list is [], retry=0 2024-11-22T23:43:41,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45113,DS-ffab1390-4579-4e6e-a937-5f1ca90e7ead,DISK] 2024-11-22T23:43:41,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T23:43:41,945 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/WALs/a511e45b3024,41373,1732319019939/a511e45b3024%2C41373%2C1732319019939.1732319021874 2024-11-22T23:43:41,946 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37403:37403)] 2024-11-22T23:43:41,946 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:43:41,946 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:41,950 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:41,950 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:41,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T23:43:42,016 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:42,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:42,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T23:43:42,022 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:42,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:42,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,025 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T23:43:42,025 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:42,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:42,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,029 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T23:43:42,029 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:42,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:42,030 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,033 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,034 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,039 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,039 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,042 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:43:42,045 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:43:42,049 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:42,050 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63747441, jitterRate=-0.05008910596370697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:43:42,055 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732319021962Initializing all the Stores at 1732319021964 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319021965 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319021965Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319021966 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319021966Cleaning up temporary data from old regions at 1732319022039 (+73 ms)Region opened successfully at 1732319022055 (+16 ms) 2024-11-22T23:43:42,057 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T23:43:42,089 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45378201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:43:42,120 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T23:43:42,131 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:42,132 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:42,135 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:42,136 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:42,141 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:42,141 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T23:43:42,172 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T23:43:42,180 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T23:43:42,182 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T23:43:42,184 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T23:43:42,186 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T23:43:42,187 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T23:43:42,190 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T23:43:42,193 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T23:43:42,195 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T23:43:42,196 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T23:43:42,199 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T23:43:42,218 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T23:43:42,219 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T23:43:42,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:43:42,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:43:42,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:42,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:42,226 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a511e45b3024,41373,1732319019939, sessionid=0x10142106af00000, setting cluster-up flag (Was=false) 2024-11-22T23:43:42,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:42,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:42,247 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T23:43:42,249 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,41373,1732319019939 2024-11-22T23:43:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:42,264 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T23:43:42,265 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,41373,1732319019939 2024-11-22T23:43:42,272 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T23:43:42,318 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(746): ClusterId : e9b0a6ea-23d3-4fcd-89f5-5129f7df5b64 2024-11-22T23:43:42,321 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:43:42,325 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:43:42,326 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:43:42,328 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:43:42,329 DEBUG [RS:0;a511e45b3024:43005 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c59174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:43:42,343 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a511e45b3024:43005 2024-11-22T23:43:42,346 INFO [RS:0;a511e45b3024:43005 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:43:42,346 INFO [RS:0;a511e45b3024:43005 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:43:42,346 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:43:42,348 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,41373,1732319019939 with port=43005, startcode=1732319020630 2024-11-22T23:43:42,351 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:42,359 DEBUG [RS:0;a511e45b3024:43005 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:43:42,362 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T23:43:42,371 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T23:43:42,378 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a511e45b3024,41373,1732319019939 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T23:43:42,386 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:43:42,386 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:43:42,386 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:43:42,387 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:43:42,387 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a511e45b3024:0, corePoolSize=10, maxPoolSize=10 2024-11-22T23:43:42,387 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,387 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:43:42,387 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,390 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319052390 2024-11-22T23:43:42,390 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33349, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:43:42,391 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T23:43:42,392 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T23:43:42,394 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:42,395 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T23:43:42,397 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:42,397 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T23:43:42,397 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T23:43:42,398 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T23:43:42,398 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T23:43:42,401 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:42,401 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T23:43:42,399 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,405 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T23:43:42,407 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T23:43:42,407 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T23:43:42,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741831_1007 (size=1321) 2024-11-22T23:43:42,413 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T23:43:42,414 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T23:43:42,415 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319022415,5,FailOnTimeoutGroup] 2024-11-22T23:43:42,418 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319022415,5,FailOnTimeoutGroup] 2024-11-22T23:43:42,418 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,419 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T23:43:42,420 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,420 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,424 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T23:43:42,424 WARN [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T23:43:42,525 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,41373,1732319019939 with port=43005, startcode=1732319020630 2024-11-22T23:43:42,528 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,43005,1732319020630 2024-11-22T23:43:42,530 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,43005,1732319020630 2024-11-22T23:43:42,538 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:42,539 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35361 2024-11-22T23:43:42,539 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:43:42,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:43:42,543 DEBUG [RS:0;a511e45b3024:43005 {}] zookeeper.ZKUtil(111): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,43005,1732319020630 2024-11-22T23:43:42,544 WARN [RS:0;a511e45b3024:43005 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:43:42,544 INFO [RS:0;a511e45b3024:43005 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:43:42,544 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630 2024-11-22T23:43:42,546 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,43005,1732319020630] 2024-11-22T23:43:42,569 INFO [RS:0;a511e45b3024:43005 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:43:42,581 INFO [RS:0;a511e45b3024:43005 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:43:42,585 INFO [RS:0;a511e45b3024:43005 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:43:42,586 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,586 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:43:42,592 INFO [RS:0;a511e45b3024:43005 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:43:42,593 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,593 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,594 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,595 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,595 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,595 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:43:42,595 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:43:42,595 DEBUG [RS:0;a511e45b3024:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:43:42,596 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,596 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,597 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,597 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,597 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,597 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,43005,1732319020630-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:43:42,617 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:43:42,619 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,43005,1732319020630-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,619 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,619 INFO [RS:0;a511e45b3024:43005 {}] regionserver.Replication(171): a511e45b3024,43005,1732319020630 started 2024-11-22T23:43:42,638 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:42,638 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,43005,1732319020630, RpcServer on a511e45b3024/172.17.0.2:43005, sessionid=0x10142106af00001 2024-11-22T23:43:42,639 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:43:42,639 DEBUG [RS:0;a511e45b3024:43005 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,43005,1732319020630 2024-11-22T23:43:42,639 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,43005,1732319020630' 2024-11-22T23:43:42,639 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:43:42,640 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:43:42,641 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:43:42,641 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:43:42,641 DEBUG [RS:0;a511e45b3024:43005 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,43005,1732319020630 2024-11-22T23:43:42,641 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,43005,1732319020630' 2024-11-22T23:43:42,641 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:43:42,642 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:43:42,642 DEBUG [RS:0;a511e45b3024:43005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:43:42,643 INFO [RS:0;a511e45b3024:43005 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:43:42,643 INFO [RS:0;a511e45b3024:43005 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:43:42,748 INFO [RS:0;a511e45b3024:43005 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:43:42,751 INFO [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C43005%2C1732319020630, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/oldWALs, maxLogs=32 2024-11-22T23:43:42,768 DEBUG [RS:0;a511e45b3024:43005 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630/a511e45b3024%2C43005%2C1732319020630.1732319022754, exclude list is [], retry=0 2024-11-22T23:43:42,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45113,DS-ffab1390-4579-4e6e-a937-5f1ca90e7ead,DISK] 2024-11-22T23:43:42,777 INFO [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630/a511e45b3024%2C43005%2C1732319020630.1732319022754 2024-11-22T23:43:42,778 DEBUG [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37403:37403)] 2024-11-22T23:43:42,813 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T23:43:42,814 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:42,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741833_1009 (size=32) 2024-11-22T23:43:43,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:43,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T23:43:43,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:43:43,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T23:43:43,233 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:43:43,234 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T23:43:43,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:43:43,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T23:43:43,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:43:43,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:43:43,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740 2024-11-22T23:43:43,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740 2024-11-22T23:43:43,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:43:43,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:43:43,247 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:43:43,249 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:43:43,253 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:43,254 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62126658, jitterRate=-0.07424065470695496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:43:43,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732319023225Initializing all the Stores at 1732319023227 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319023227Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319023227Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319023227Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319023227Cleaning up temporary data from old regions at 1732319023246 (+19 ms)Region opened successfully at 1732319023256 (+10 ms) 2024-11-22T23:43:43,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:43:43,256 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:43:43,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:43:43,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:43:43,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:43:43,258 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:43:43,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319023256Disabling compacts and flushes for region at 1732319023256Disabling writes for close at 1732319023256Writing region close event to WAL at 1732319023257 (+1 ms)Closed at 1732319023258 (+1 ms) 2024-11-22T23:43:43,261 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:43,261 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T23:43:43,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:43:43,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:43:43,280 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T23:43:43,433 DEBUG [a511e45b3024:41373 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T23:43:43,445 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:43,452 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPENING 2024-11-22T23:43:43,459 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T23:43:43,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:43,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:43,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:43,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:43,463 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:43:43,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:43,641 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:43:43,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:43:43,655 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:43:43,655 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:43:43,656 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:43:43,659 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C43005%2C1732319020630.meta, suffix=.meta, logDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/oldWALs, maxLogs=32 2024-11-22T23:43:43,675 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630/a511e45b3024%2C43005%2C1732319020630.meta.1732319023660.meta, exclude list is [], retry=0 2024-11-22T23:43:43,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45113,DS-ffab1390-4579-4e6e-a937-5f1ca90e7ead,DISK] 2024-11-22T23:43:43,682 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630/a511e45b3024%2C43005%2C1732319020630.meta.1732319023660.meta 2024-11-22T23:43:43,683 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37403:37403)] 2024-11-22T23:43:43,683 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:43:43,685 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:43:43,688 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:43:43,693 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:43:43,697 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:43:43,698 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:43,698 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:43:43,698 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:43:43,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T23:43:43,703 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:43:43,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T23:43:43,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:43:43,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T23:43:43,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:43:43,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T23:43:43,711 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:43:43,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:43,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:43:43,712 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:43:43,713 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740 2024-11-22T23:43:43,716 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740 2024-11-22T23:43:43,718 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:43:43,718 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:43:43,719 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:43:43,722 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:43:43,723 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74631614, jitterRate=0.11209771037101746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:43:43,724 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:43:43,725 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732319023699Writing region info on filesystem at 1732319023699Initializing all the Stores at 1732319023701 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319023701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319023701Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319023701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732319023701Cleaning up temporary data from old regions at 1732319023718 (+17 ms)Running coprocessor post-open hooks at 1732319023724 (+6 ms)Region opened successfully at 1732319023725 (+1 ms) 2024-11-22T23:43:43,732 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732319023631 2024-11-22T23:43:43,743 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:43:43,744 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:43:43,746 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:43,748 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:43,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:43,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:43,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:43,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:43,758 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:43,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T23:43:43,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,43005,1732319020630 in 293 msec 2024-11-22T23:43:43,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T23:43:43,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 499 msec 2024-11-22T23:43:43,773 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:43,773 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T23:43:43,794 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:43:43,796 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,43005,1732319020630, seqNum=-1] 2024-11-22T23:43:43,818 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:43:43,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44821, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:43:43,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5470 sec 2024-11-22T23:43:43,849 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732319023849, completionTime=-1 2024-11-22T23:43:43,852 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T23:43:43,852 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:43,877 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:43,877 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319083877 2024-11-22T23:43:43,877 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319143877 2024-11-22T23:43:43,877 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-22T23:43:43,879 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41373,1732319019939-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,880 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41373,1732319019939-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,880 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41373,1732319019939-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,881 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a511e45b3024:41373, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,882 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,883 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,889 DEBUG [master/a511e45b3024:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T23:43:43,909 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.138sec 2024-11-22T23:43:43,911 INFO [master/a511e45b3024:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T23:43:43,912 INFO [master/a511e45b3024:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T23:43:43,913 INFO [master/a511e45b3024:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T23:43:43,913 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T23:43:43,913 INFO [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T23:43:43,914 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41373,1732319019939-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:43:43,915 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41373,1732319019939-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T23:43:43,923 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T23:43:43,924 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T23:43:43,924 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,41373,1732319019939-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:43:43,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371ae7bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:43:43,950 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T23:43:43,950 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T23:43:43,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,41373,-1 for getting cluster id 2024-11-22T23:43:43,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:43:43,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e9b0a6ea-23d3-4fcd-89f5-5129f7df5b64' 2024-11-22T23:43:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:43:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e9b0a6ea-23d3-4fcd-89f5-5129f7df5b64" 2024-11-22T23:43:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d055ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:43:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,41373,-1] 2024-11-22T23:43:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:43:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:43,976 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:43:43,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9637c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:43:43,980 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:43:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,43005,1732319020630, seqNum=-1] 2024-11-22T23:43:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:43:43,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:43:44,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a511e45b3024,41373,1732319019939 2024-11-22T23:43:44,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTable Thread=207, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=8964 2024-11-22T23:43:44,027 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T23:43:44,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:44,039 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:43:44,042 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:44,044 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:43:44,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741835_1011 (size=639) 2024-11-22T23:43:44,460 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 790878d3607ba77e156cb1f08c836022, NAME => 'testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:44,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741836_1012 (size=51) 2024-11-22T23:43:44,870 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:44,870 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1722): Closing 790878d3607ba77e156cb1f08c836022, disabling compactions & flushes 2024-11-22T23:43:44,870 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:44,870 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:44,870 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. after waiting 0 ms 2024-11-22T23:43:44,870 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:44,870 INFO [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:44,870 DEBUG [RegionOpenAndInit-testDisableTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 790878d3607ba77e156cb1f08c836022: Waiting for close lock at 1732319024870Disabling compacts and flushes for region at 1732319024870Disabling writes for close at 1732319024870Writing region close event to WAL at 1732319024870Closed at 1732319024870 2024-11-22T23:43:44,873 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:43:44,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1732319024873"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319024873"}]},"ts":"1732319024873"} 2024-11-22T23:43:44,883 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:43:44,884 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:43:44,887 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319024885"}]},"ts":"1732319024885"} 2024-11-22T23:43:44,891 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLING in hbase:meta 2024-11-22T23:43:44,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, ASSIGN}] 2024-11-22T23:43:44,896 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, ASSIGN 2024-11-22T23:43:44,898 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, ASSIGN; state=OFFLINE, location=a511e45b3024,43005,1732319020630; forceNewPlan=false, retain=false 2024-11-22T23:43:45,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=790878d3607ba77e156cb1f08c836022, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:45,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, ASSIGN because future has completed 2024-11-22T23:43:45,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 790878d3607ba77e156cb1f08c836022, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:45,216 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,217 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 790878d3607ba77e156cb1f08c836022, NAME => 'testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:43:45,217 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTable 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,217 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:45,217 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,218 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,220 INFO [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,222 INFO [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 790878d3607ba77e156cb1f08c836022 columnFamilyName f1 2024-11-22T23:43:45,223 DEBUG [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:45,224 INFO [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] regionserver.HStore(327): Store=790878d3607ba77e156cb1f08c836022/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:45,224 INFO [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,226 INFO [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 790878d3607ba77e156cb1f08c836022 columnFamilyName f2 2024-11-22T23:43:45,226 DEBUG [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:45,227 INFO [StoreOpener-790878d3607ba77e156cb1f08c836022-1 {}] regionserver.HStore(327): Store=790878d3607ba77e156cb1f08c836022/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:45,227 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,228 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,229 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,230 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,230 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,231 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTable descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T23:43:45,233 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,236 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:45,237 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 790878d3607ba77e156cb1f08c836022; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74902625, jitterRate=0.1161360889673233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T23:43:45,237 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,238 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 790878d3607ba77e156cb1f08c836022: Running coprocessor pre-open hook at 1732319025218Writing region info on filesystem at 1732319025218Initializing all the Stores at 1732319025219 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319025219Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319025220 (+1 ms)Cleaning up temporary data from old regions at 1732319025230 (+10 ms)Running coprocessor post-open hooks at 1732319025237 (+7 ms)Region opened successfully at 1732319025238 (+1 ms) 2024-11-22T23:43:45,240 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022., pid=6, masterSystemTime=1732319025209 2024-11-22T23:43:45,244 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,244 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,246 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=790878d3607ba77e156cb1f08c836022, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:45,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 790878d3607ba77e156cb1f08c836022, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:45,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T23:43:45,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 790878d3607ba77e156cb1f08c836022, server=a511e45b3024,43005,1732319020630 in 196 msec 2024-11-22T23:43:45,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T23:43:45,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, ASSIGN in 362 msec 2024-11-22T23:43:45,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:43:45,261 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319025261"}]},"ts":"1732319025261"} 2024-11-22T23:43:45,265 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=ENABLED in hbase:meta 2024-11-22T23:43:45,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTable execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:43:45,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable in 1.2360 sec 2024-11-22T23:43:45,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:45,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319025298"}]},"ts":"1732319025298"} 2024-11-22T23:43:45,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLING in hbase:meta 2024-11-22T23:43:45,302 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTable to state=DISABLING 2024-11-22T23:43:45,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTable}] 2024-11-22T23:43:45,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, UNASSIGN}] 2024-11-22T23:43:45,312 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, UNASSIGN 2024-11-22T23:43:45,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=790878d3607ba77e156cb1f08c836022, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:45,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, UNASSIGN because future has completed 2024-11-22T23:43:45,318 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:45,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 790878d3607ba77e156cb1f08c836022, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:45,477 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,477 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:45,478 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 790878d3607ba77e156cb1f08c836022, disabling compactions & flushes 2024-11-22T23:43:45,478 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,478 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,478 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. after waiting 0 ms 2024-11-22T23:43:45,478 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,485 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:43:45,487 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022. 2024-11-22T23:43:45,487 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 790878d3607ba77e156cb1f08c836022: Waiting for close lock at 1732319025477Running coprocessor pre-close hooks at 1732319025477Disabling compacts and flushes for region at 1732319025477Disabling writes for close at 1732319025478 (+1 ms)Writing region close event to WAL at 1732319025478Running coprocessor post-close hooks at 1732319025486 (+8 ms)Closed at 1732319025487 (+1 ms) 2024-11-22T23:43:45,491 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,492 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=790878d3607ba77e156cb1f08c836022, regionState=CLOSED 2024-11-22T23:43:45,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 790878d3607ba77e156cb1f08c836022, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:45,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T23:43:45,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 790878d3607ba77e156cb1f08c836022, server=a511e45b3024,43005,1732319020630 in 179 msec 2024-11-22T23:43:45,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T23:43:45,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTable, region=790878d3607ba77e156cb1f08c836022, UNASSIGN in 190 msec 2024-11-22T23:43:45,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T23:43:45,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTable in 200 msec 2024-11-22T23:43:45,509 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319025509"}]},"ts":"1732319025509"} 2024-11-22T23:43:45,512 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTable, state=DISABLED in hbase:meta 2024-11-22T23:43:45,512 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTable to state=DISABLED 2024-11-22T23:43:45,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable in 224 msec 2024-11-22T23:43:45,544 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T23:43:45,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:43:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,41373,1732319019939 2024-11-22T23:43:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1aaf840e 2024-11-22T23:43:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:43:45,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:43:45,560 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTable 2024-11-22T23:43:45,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTable 2024-11-22T23:43:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] ipc.CallRunner(138): callId: 2 service: MasterService methodName: DisableTable size: 77 connection: 172.17.0.2:52690 deadline: 1732319085563, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable 2024-11-22T23:43:45,569 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTable failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTable at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:45,569 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTable already disabled, so just deleting it. 2024-11-22T23:43:45,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTable 2024-11-22T23:43:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] procedure2.ProcedureExecutor(1139): Stored pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:45,579 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=12, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:45,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=12, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-22T23:43:45,588 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,593 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/f1, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/f2, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/recovered.edits] 2024-11-22T23:43:45,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/recovered.edits/4.seqid to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/archive/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022/recovered.edits/4.seqid 2024-11-22T23:43:45,606 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTable/790878d3607ba77e156cb1f08c836022 2024-11-22T23:43:45,606 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTable regions 2024-11-22T23:43:45,612 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=12, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:45,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T23:43:45,622 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTable from hbase:meta 2024-11-22T23:43:45,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTable' descriptor. 2024-11-22T23:43:45,657 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=12, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:45,657 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTable' from region states. 2024-11-22T23:43:45,658 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319025657"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:45,662 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:43:45,663 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 790878d3607ba77e156cb1f08c836022, NAME => 'testDisableTable,,1732319024030.790878d3607ba77e156cb1f08c836022.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:43:45,663 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTable' as deleted. 2024-11-22T23:43:45,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTable","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732319025663"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:45,666 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTable state from META 2024-11-22T23:43:45,668 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=12, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:45,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable in 95 msec 2024-11-22T23:43:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=12 2024-11-22T23:43:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTable 2024-11-22T23:43:45,695 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTable completed 2024-11-22T23:43:45,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTable Thread=217 (was 207) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2126914916_22 at /127.0.0.1:44378 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a511e45b3024:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 292), ProcessCount=11 (was 11), AvailableMemoryMB=8943 (was 8964) 2024-11-22T23:43:45,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=217, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=8943 2024-11-22T23:43:45,714 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T23:43:45,717 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:45,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:43:45,719 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:45,720 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:43:45,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741837_1013 (size=652) 2024-11-22T23:43:46,132 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f76b4ec611f3dadcac6fb8ac461d2863, NAME => 'testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDisableTableMultipleTimes', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741838_1014 (size=64) 2024-11-22T23:43:46,147 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:46,147 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1722): Closing f76b4ec611f3dadcac6fb8ac461d2863, disabling compactions & flushes 2024-11-22T23:43:46,147 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,147 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,147 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. after waiting 0 ms 2024-11-22T23:43:46,147 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,147 INFO [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,147 DEBUG [RegionOpenAndInit-testDisableTableMultipleTimes-pool-0 {}] regionserver.HRegion(1676): Region close journal for f76b4ec611f3dadcac6fb8ac461d2863: Waiting for close lock at 1732319026147Disabling compacts and flushes for region at 1732319026147Disabling writes for close at 1732319026147Writing region close event to WAL at 1732319026147Closed at 1732319026147 2024-11-22T23:43:46,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:43:46,150 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732319026149"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319026149"}]},"ts":"1732319026149"} 2024-11-22T23:43:46,153 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:43:46,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:43:46,155 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319026155"}]},"ts":"1732319026155"} 2024-11-22T23:43:46,158 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLING in hbase:meta 2024-11-22T23:43:46,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, ASSIGN}] 2024-11-22T23:43:46,161 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, ASSIGN 2024-11-22T23:43:46,162 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, ASSIGN; state=OFFLINE, location=a511e45b3024,43005,1732319020630; forceNewPlan=false, retain=false 2024-11-22T23:43:46,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f76b4ec611f3dadcac6fb8ac461d2863, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:46,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, ASSIGN because future has completed 2024-11-22T23:43:46,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f76b4ec611f3dadcac6fb8ac461d2863, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:46,476 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,476 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => f76b4ec611f3dadcac6fb8ac461d2863, NAME => 'testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:43:46,477 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testDisableTableMultipleTimes f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,477 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:46,477 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,478 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,480 INFO [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,482 INFO [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f76b4ec611f3dadcac6fb8ac461d2863 columnFamilyName f1 2024-11-22T23:43:46,482 DEBUG [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:46,483 INFO [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] regionserver.HStore(327): Store=f76b4ec611f3dadcac6fb8ac461d2863/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:46,483 INFO [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,486 INFO [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f76b4ec611f3dadcac6fb8ac461d2863 columnFamilyName f2 2024-11-22T23:43:46,486 DEBUG [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:46,487 INFO [StoreOpener-f76b4ec611f3dadcac6fb8ac461d2863-1 {}] regionserver.HStore(327): Store=f76b4ec611f3dadcac6fb8ac461d2863/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:46,487 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,488 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,489 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,490 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,490 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,491 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testDisableTableMultipleTimes descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T23:43:46,493 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,496 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:46,497 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened f76b4ec611f3dadcac6fb8ac461d2863; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71169204, jitterRate=0.06050378084182739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T23:43:46,497 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,498 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for f76b4ec611f3dadcac6fb8ac461d2863: Running coprocessor pre-open hook at 1732319026478Writing region info on filesystem at 1732319026478Initializing all the Stores at 1732319026479 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319026479Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319026480 (+1 ms)Cleaning up temporary data from old regions at 1732319026490 (+10 ms)Running coprocessor post-open hooks at 1732319026497 (+7 ms)Region opened successfully at 1732319026498 (+1 ms) 2024-11-22T23:43:46,499 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863., pid=15, masterSystemTime=1732319026470 2024-11-22T23:43:46,502 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,502 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f76b4ec611f3dadcac6fb8ac461d2863, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:46,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f76b4ec611f3dadcac6fb8ac461d2863, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:46,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-22T23:43:46,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure f76b4ec611f3dadcac6fb8ac461d2863, server=a511e45b3024,43005,1732319020630 in 191 msec 2024-11-22T23:43:46,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T23:43:46,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, ASSIGN in 353 msec 2024-11-22T23:43:46,515 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:43:46,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319026515"}]},"ts":"1732319026515"} 2024-11-22T23:43:46,518 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=ENABLED in hbase:meta 2024-11-22T23:43:46,519 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testDisableTableMultipleTimes execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:43:46,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes in 804 msec 2024-11-22T23:43:46,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:46,723 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319026723"}]},"ts":"1732319026723"} 2024-11-22T23:43:46,725 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLING in hbase:meta 2024-11-22T23:43:46,725 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testDisableTableMultipleTimes to state=DISABLING 2024-11-22T23:43:46,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes}] 2024-11-22T23:43:46,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, UNASSIGN}] 2024-11-22T23:43:46,732 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, UNASSIGN 2024-11-22T23:43:46,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f76b4ec611f3dadcac6fb8ac461d2863, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:46,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, UNASSIGN because future has completed 2024-11-22T23:43:46,736 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:46,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure f76b4ec611f3dadcac6fb8ac461d2863, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:46,890 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(122): Close f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,890 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:46,890 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1722): Closing f76b4ec611f3dadcac6fb8ac461d2863, disabling compactions & flushes 2024-11-22T23:43:46,890 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1755): Closing region testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,891 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1776): Time limited wait for close lock on testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,891 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1843): Acquired close lock on testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. after waiting 0 ms 2024-11-22T23:43:46,891 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1853): Updates disabled for region testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,897 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:43:46,898 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1973): Closed testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863. 2024-11-22T23:43:46,898 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] regionserver.HRegion(1676): Region close journal for f76b4ec611f3dadcac6fb8ac461d2863: Waiting for close lock at 1732319026890Running coprocessor pre-close hooks at 1732319026890Disabling compacts and flushes for region at 1732319026890Disabling writes for close at 1732319026891 (+1 ms)Writing region close event to WAL at 1732319026891Running coprocessor post-close hooks at 1732319026898 (+7 ms)Closed at 1732319026898 2024-11-22T23:43:46,901 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=19}] handler.UnassignRegionHandler(157): Closed f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f76b4ec611f3dadcac6fb8ac461d2863, regionState=CLOSED 2024-11-22T23:43:46,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; CloseRegionProcedure f76b4ec611f3dadcac6fb8ac461d2863, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:46,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-22T23:43:46,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; CloseRegionProcedure f76b4ec611f3dadcac6fb8ac461d2863, server=a511e45b3024,43005,1732319020630 in 170 msec 2024-11-22T23:43:46,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T23:43:46,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testDisableTableMultipleTimes, region=f76b4ec611f3dadcac6fb8ac461d2863, UNASSIGN in 179 msec 2024-11-22T23:43:46,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-22T23:43:46,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testDisableTableMultipleTimes in 187 msec 2024-11-22T23:43:46,917 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319026917"}]},"ts":"1732319026917"} 2024-11-22T23:43:46,920 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testDisableTableMultipleTimes, state=DISABLED in hbase:meta 2024-11-22T23:43:46,920 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testDisableTableMultipleTimes to state=DISABLED 2024-11-22T23:43:46,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes in 204 msec 2024-11-22T23:43:46,970 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(101): Disable failed with exception {}org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes 2024-11-22T23:43:46,970 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(114): Disable failed with expected exception {} org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:109) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:46,971 DEBUG [Time-limited test {}] procedure.TestDisableTableProcedure(126): Disable failed with expected exception {} org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:64) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.TestDisableTableProcedure.testDisableTableMultipleTimes(TestDisableTableProcedure.java:120) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:46,971 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T23:43:46,973 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testDisableTableMultipleTimes 2024-11-22T23:43:46,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testDisableTableMultipleTimes 2024-11-22T23:43:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] ipc.CallRunner(138): callId: 7 service: MasterService methodName: DisableTable size: 90 connection: 172.17.0.2:52690 deadline: 1732319086974, exception=org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes 2024-11-22T23:43:46,975 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testDisableTableMultipleTimes failed with org.apache.hadoop.hbase.TableNotEnabledException: testDisableTableMultipleTimes at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:46,975 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testDisableTableMultipleTimes already disabled, so just deleting it. 2024-11-22T23:43:46,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testDisableTableMultipleTimes 2024-11-22T23:43:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:46,979 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=21, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:43:46,980 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=21, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:46,984 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,987 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/f1, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/f2, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/recovered.edits] 2024-11-22T23:43:46,997 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/recovered.edits/4.seqid to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/archive/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863/recovered.edits/4.seqid 2024-11-22T23:43:46,997 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testDisableTableMultipleTimes/f76b4ec611f3dadcac6fb8ac461d2863 2024-11-22T23:43:46,997 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testDisableTableMultipleTimes regions 2024-11-22T23:43:47,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=21, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:47,005 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of testDisableTableMultipleTimes from hbase:meta 2024-11-22T23:43:47,008 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testDisableTableMultipleTimes' descriptor. 2024-11-22T23:43:47,010 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=21, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:47,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testDisableTableMultipleTimes' from region states. 2024-11-22T23:43:47,011 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319027011"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:47,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:43:47,014 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f76b4ec611f3dadcac6fb8ac461d2863, NAME => 'testDisableTableMultipleTimes,,1732319025715.f76b4ec611f3dadcac6fb8ac461d2863.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:43:47,014 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testDisableTableMultipleTimes' as deleted. 2024-11-22T23:43:47,014 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testDisableTableMultipleTimes","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732319027014"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:47,017 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testDisableTableMultipleTimes state from META 2024-11-22T23:43:47,018 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=21, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:47,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes in 42 msec 2024-11-22T23:43:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:43:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testDisableTableMultipleTimes 2024-11-22T23:43:47,086 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testDisableTableMultipleTimes completed 2024-11-22T23:43:47,095 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testDisableTableMultipleTimes Thread=222 (was 217) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2126914916_22 at /127.0.0.1:44378 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=8936 (was 8943) 2024-11-22T23:43:47,103 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=8935 2024-11-22T23:43:47,103 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T23:43:47,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:47,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:43:47,107 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:47,109 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:43:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741839_1015 (size=653) 2024-11-22T23:43:47,520 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b75b8feae225dafecff6deed65f9c1cc, NAME => 'testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.', STARTKEY => '', ENDKEY => 'a'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:47,521 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6ea6628446fa75243da49391aac996c2, NAME => 'testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.', STARTKEY => 'b', ENDKEY => 'c'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:47,521 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 287e1377df0169cd54d478142b508d46, NAME => 'testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.', STARTKEY => 'a', ENDKEY => 'b'}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:47,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741841_1017 (size=66) 2024-11-22T23:43:47,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741840_1016 (size=67) 2024-11-22T23:43:47,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741842_1018 (size=67) 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 287e1377df0169cd54d478142b508d46, disabling compactions & flushes 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1722): Closing 6ea6628446fa75243da49391aac996c2, disabling compactions & flushes 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1722): Closing b75b8feae225dafecff6deed65f9c1cc, disabling compactions & flushes 2024-11-22T23:43:47,933 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:47,933 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:47,933 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:47,933 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. after waiting 0 ms 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:47,934 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. after waiting 0 ms 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. after waiting 0 ms 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 287e1377df0169cd54d478142b508d46: Waiting for close lock at 1732319027933Disabling compacts and flushes for region at 1732319027933Disabling writes for close at 1732319027934 (+1 ms)Writing region close event to WAL at 1732319027934Closed at 1732319027934 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:47,934 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:47,934 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-0 {}] regionserver.HRegion(1676): Region close journal for b75b8feae225dafecff6deed65f9c1cc: Waiting for close lock at 1732319027933Disabling compacts and flushes for region at 1732319027933Disabling writes for close at 1732319027934 (+1 ms)Writing region close event to WAL at 1732319027934Closed at 1732319027934 2024-11-22T23:43:47,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6ea6628446fa75243da49391aac996c2: Waiting for close lock at 1732319027933Disabling compacts and flushes for region at 1732319027933Disabling writes for close at 1732319027934 (+1 ms)Writing region close event to WAL at 1732319027934Closed at 1732319027934 2024-11-22T23:43:47,934 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1bfcae741ae1bdf7847ea9d7f176cbb4, NAME => 'testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.', STARTKEY => 'c', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc 2024-11-22T23:43:47,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741843_1019 (size=66) 2024-11-22T23:43:48,343 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:48,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1722): Closing 1bfcae741ae1bdf7847ea9d7f176cbb4, disabling compactions & flushes 2024-11-22T23:43:48,344 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. after waiting 0 ms 2024-11-22T23:43:48,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,344 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1bfcae741ae1bdf7847ea9d7f176cbb4: Waiting for close lock at 1732319028344Disabling compacts and flushes for region at 1732319028344Disabling writes for close at 1732319028344Writing region close event to WAL at 1732319028344Closed at 1732319028344 2024-11-22T23:43:48,346 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:43:48,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732319028346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319028346"}]},"ts":"1732319028346"} 2024-11-22T23:43:48,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1732319028346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319028346"}]},"ts":"1732319028346"} 2024-11-22T23:43:48,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732319028346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319028346"}]},"ts":"1732319028346"} 2024-11-22T23:43:48,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1732319028346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732319028346"}]},"ts":"1732319028346"} 2024-11-22T23:43:48,352 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 4 regions to meta. 2024-11-22T23:43:48,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:43:48,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319028353"}]},"ts":"1732319028353"} 2024-11-22T23:43:48,356 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLING in hbase:meta 2024-11-22T23:43:48,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, ASSIGN}, {pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, ASSIGN}, {pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, ASSIGN}, {pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, ASSIGN}] 2024-11-22T23:43:48,359 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, ASSIGN 2024-11-22T23:43:48,360 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, ASSIGN; state=OFFLINE, location=a511e45b3024,43005,1732319020630; forceNewPlan=false, retain=false 2024-11-22T23:43:48,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, ASSIGN 2024-11-22T23:43:48,362 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, ASSIGN; state=OFFLINE, location=a511e45b3024,43005,1732319020630; forceNewPlan=false, retain=false 2024-11-22T23:43:48,363 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, ASSIGN 2024-11-22T23:43:48,364 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, ASSIGN; state=OFFLINE, location=a511e45b3024,43005,1732319020630; forceNewPlan=false, retain=false 2024-11-22T23:43:48,365 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, ASSIGN 2024-11-22T23:43:48,366 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, ASSIGN; state=OFFLINE, location=a511e45b3024,43005,1732319020630; forceNewPlan=false, retain=false 2024-11-22T23:43:48,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b75b8feae225dafecff6deed65f9c1cc, regionState=OPENING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, ASSIGN because future has completed 2024-11-22T23:43:48,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:48,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, ASSIGN because future has completed 2024-11-22T23:43:48,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, ASSIGN because future has completed 2024-11-22T23:43:48,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, ASSIGN because future has completed 2024-11-22T23:43:48,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:48,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:48,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:48,564 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T23:43:48,673 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,673 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bfcae741ae1bdf7847ea9d7f176cbb4, NAME => 'testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.', STARTKEY => 'c', ENDKEY => ''} 2024-11-22T23:43:48,674 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,674 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:48,674 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,674 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,676 INFO [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,678 INFO [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bfcae741ae1bdf7847ea9d7f176cbb4 columnFamilyName f1 2024-11-22T23:43:48,678 DEBUG [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,679 INFO [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] regionserver.HStore(327): Store=1bfcae741ae1bdf7847ea9d7f176cbb4/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,679 INFO [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,680 INFO [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bfcae741ae1bdf7847ea9d7f176cbb4 columnFamilyName f2 2024-11-22T23:43:48,681 DEBUG [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,681 INFO [StoreOpener-1bfcae741ae1bdf7847ea9d7f176cbb4-1 {}] regionserver.HStore(327): Store=1bfcae741ae1bdf7847ea9d7f176cbb4/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,681 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,682 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,683 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,684 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,684 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,685 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T23:43:48,686 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,689 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:48,690 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1114): Opened 1bfcae741ae1bdf7847ea9d7f176cbb4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72694674, jitterRate=0.08323505520820618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T23:43:48,690 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:48,691 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 1bfcae741ae1bdf7847ea9d7f176cbb4: Running coprocessor pre-open hook at 1732319028674Writing region info on filesystem at 1732319028674Initializing all the Stores at 1732319028675 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028676 (+1 ms)Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028676Cleaning up temporary data from old regions at 1732319028684 (+8 ms)Running coprocessor post-open hooks at 1732319028690 (+6 ms)Region opened successfully at 1732319028691 (+1 ms) 2024-11-22T23:43:48,692 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4., pid=27, masterSystemTime=1732319028667 2024-11-22T23:43:48,694 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,695 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=27}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:48,695 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:48,695 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => b75b8feae225dafecff6deed65f9c1cc, NAME => 'testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.', STARTKEY => '', ENDKEY => 'a'} 2024-11-22T23:43:48,695 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,695 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:48,695 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,695 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,695 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,698 INFO [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:48,699 INFO [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b75b8feae225dafecff6deed65f9c1cc columnFamilyName f1 2024-11-22T23:43:48,700 DEBUG [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,700 INFO [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] regionserver.HStore(327): Store=b75b8feae225dafecff6deed65f9c1cc/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,701 INFO [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,702 INFO [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b75b8feae225dafecff6deed65f9c1cc columnFamilyName f2 2024-11-22T23:43:48,702 DEBUG [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-22T23:43:48,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 in 185 msec 2024-11-22T23:43:48,703 INFO [StoreOpener-b75b8feae225dafecff6deed65f9c1cc-1 {}] regionserver.HStore(327): Store=b75b8feae225dafecff6deed65f9c1cc/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,703 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,704 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, ASSIGN in 347 msec 2024-11-22T23:43:48,705 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,706 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,706 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,707 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T23:43:48,708 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,711 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:48,711 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened b75b8feae225dafecff6deed65f9c1cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70933539, jitterRate=0.056992098689079285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T23:43:48,712 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:48,712 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for b75b8feae225dafecff6deed65f9c1cc: Running coprocessor pre-open hook at 1732319028696Writing region info on filesystem at 1732319028696Initializing all the Stores at 1732319028697 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028697Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028697Cleaning up temporary data from old regions at 1732319028706 (+9 ms)Running coprocessor post-open hooks at 1732319028712 (+6 ms)Region opened successfully at 1732319028712 2024-11-22T23:43:48,713 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc., pid=28, masterSystemTime=1732319028667 2024-11-22T23:43:48,715 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:48,716 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:48,716 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:48,716 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ea6628446fa75243da49391aac996c2, NAME => 'testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.', STARTKEY => 'b', ENDKEY => 'c'} 2024-11-22T23:43:48,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,716 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,716 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:48,717 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,717 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,719 INFO [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:48,720 INFO [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea6628446fa75243da49391aac996c2 columnFamilyName f1 2024-11-22T23:43:48,721 DEBUG [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,721 INFO [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] regionserver.HStore(327): Store=6ea6628446fa75243da49391aac996c2/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,721 INFO [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=23 2024-11-22T23:43:48,723 INFO [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea6628446fa75243da49391aac996c2 columnFamilyName f2 2024-11-22T23:43:48,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 in 201 msec 2024-11-22T23:43:48,723 DEBUG [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,724 INFO [StoreOpener-6ea6628446fa75243da49391aac996c2-1 {}] regionserver.HStore(327): Store=6ea6628446fa75243da49391aac996c2/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,724 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,725 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, ASSIGN in 367 msec 2024-11-22T23:43:48,726 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,727 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,727 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,728 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T23:43:48,729 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,732 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:48,733 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 6ea6628446fa75243da49391aac996c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73827818, jitterRate=0.10012021660804749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T23:43:48,733 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:48,733 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 6ea6628446fa75243da49391aac996c2: Running coprocessor pre-open hook at 1732319028717Writing region info on filesystem at 1732319028717Initializing all the Stores at 1732319028718 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028718Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028718Cleaning up temporary data from old regions at 1732319028727 (+9 ms)Running coprocessor post-open hooks at 1732319028733 (+6 ms)Region opened successfully at 1732319028733 2024-11-22T23:43:48,734 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2., pid=30, masterSystemTime=1732319028667 2024-11-22T23:43:48,737 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:48,737 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:48,737 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:48,737 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7752): Opening region: {ENCODED => 287e1377df0169cd54d478142b508d46, NAME => 'testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.', STARTKEY => 'a', ENDKEY => 'b'} 2024-11-22T23:43:48,738 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,738 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:43:48,738 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7794): checking encryption for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,738 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7797): checking classloading for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,740 INFO [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f1 of region 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:48,742 INFO [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 287e1377df0169cd54d478142b508d46 columnFamilyName f1 2024-11-22T23:43:48,742 DEBUG [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,743 INFO [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] regionserver.HStore(327): Store=287e1377df0169cd54d478142b508d46/f1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,743 INFO [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=25 2024-11-22T23:43:48,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 in 214 msec 2024-11-22T23:43:48,745 INFO [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 287e1377df0169cd54d478142b508d46 columnFamilyName f2 2024-11-22T23:43:48,745 DEBUG [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:48,745 INFO [StoreOpener-287e1377df0169cd54d478142b508d46-1 {}] regionserver.HStore(327): Store=287e1377df0169cd54d478142b508d46/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:43:48,746 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1038): replaying wal for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, ASSIGN in 389 msec 2024-11-22T23:43:48,747 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,747 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,749 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1048): stopping wal replay for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,749 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1060): Cleaning up temporary data for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,749 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-22T23:43:48,751 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): writing seq id for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,754 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:43:48,755 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1114): Opened 287e1377df0169cd54d478142b508d46; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62638891, jitterRate=-0.06660778820514679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-22T23:43:48,755 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:48,755 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1006): Region open journal for 287e1377df0169cd54d478142b508d46: Running coprocessor pre-open hook at 1732319028738Writing region info on filesystem at 1732319028738Initializing all the Stores at 1732319028739 (+1 ms)Instantiating store for column family {NAME => 'f1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028740 (+1 ms)Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732319028740Cleaning up temporary data from old regions at 1732319028749 (+9 ms)Running coprocessor post-open hooks at 1732319028755 (+6 ms)Region opened successfully at 1732319028755 2024-11-22T23:43:48,756 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46., pid=29, masterSystemTime=1732319028667 2024-11-22T23:43:48,758 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:48,758 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:48,759 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:48,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:48,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=24 2024-11-22T23:43:48,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 in 238 msec 2024-11-22T23:43:48,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-22T23:43:48,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, ASSIGN in 409 msec 2024-11-22T23:43:48,770 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:43:48,770 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319028770"}]},"ts":"1732319028770"} 2024-11-22T23:43:48,773 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=ENABLED in hbase:meta 2024-11-22T23:43:48,774 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:43:48,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution in 1.6700 sec 2024-11-22T23:43:48,856 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-22T23:43:48,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:48,859 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:48,859 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:48,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:48,859 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:49,108 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,109 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:49,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,109 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:49,109 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:49,109 ERROR [Time-limited test {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] 2024-11-22T23:43:49,110 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:49,110 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:49,110 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:49,110 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:49,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:49,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[] 2024-11-22T23:43:49,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:49,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:49,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:49,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:49,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,118 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:49,119 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T23:43:49,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,121 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:49,121 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:49,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,126 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319059126 2024-11-22T23:43:49,126 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:49,126 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,133 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:49,133 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319089133 2024-11-22T23:43:49,133 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319149133 2024-11-22T23:43:49,133 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-22T23:43:49,133 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:49,135 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:49,135 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,135 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:49,208 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:43:49,383 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,383 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:49,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,383 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:49,383 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:49,384 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:49,384 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:49,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:49,384 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:49,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:49,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0] 2024-11-22T23:43:49,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:49,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:49,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:49,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:49,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,392 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,392 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:49,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-22T23:43:49,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:49,394 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:49,395 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:49,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,397 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319059397 2024-11-22T23:43:49,397 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:49,397 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_PRE_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,403 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:49,403 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319089403 2024-11-22T23:43:49,403 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319149403 2024-11-22T23:43:49,403 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T23:43:49,403 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:49,405 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319029404"}]},"ts":"1732319029404"} 2024-11-22T23:43:49,407 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta 2024-11-22T23:43:49,407 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING 2024-11-22T23:43:49,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:49,407 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:49,653 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,653 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:49,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,653 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:49,653 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:49,654 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:49,654 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:49,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:49,654 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:49,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:49,657 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1] 2024-11-22T23:43:49,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:49,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:49,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:49,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:49,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:49,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:49,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,661 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:49,662 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:49,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,664 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,664 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,665 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319059665 2024-11-22T23:43:49,665 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:49,665 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLING_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,670 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:49,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319089670 2024-11-22T23:43:49,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319149670 2024-11-22T23:43:49,670 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:49,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319029670"}]},"ts":"1732319029670"} 2024-11-22T23:43:49,673 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLING in hbase:meta 2024-11-22T23:43:49,673 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set testRecoveryAndDoubleExecution to state=DISABLING 2024-11-22T23:43:49,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:49,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}] 2024-11-22T23:43:49,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:49,675 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:49,920 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,921 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:49,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,921 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:49,921 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:49,921 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:49,921 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:49,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:49,921 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:49,922 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:49,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2] 2024-11-22T23:43:49,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:49,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:49,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:49,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:49,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:49,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,927 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:49,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T23:43:49,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,928 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:49,928 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:49,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:49,930 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,930 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:49,931 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319059931 2024-11-22T23:43:49,931 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:49,931 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_MARK_REGIONS_OFFLINE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:49,936 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:49,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319089936 2024-11-22T23:43:49,936 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319149936 2024-11-22T23:43:49,936 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T23:43:49,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution}] 2024-11-22T23:43:49,936 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:49,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN}] 2024-11-22T23:43:49,940 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:49,940 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:49,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:49,940 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:50,186 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,186 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:50,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,186 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:50,187 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:50,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:50,187 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:50,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:50,187 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:50,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:50,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:50,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[] 2024-11-22T23:43:50,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:50,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:50,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:50,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:50,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T23:43:50,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,194 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:50,194 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:50,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319060198 2024-11-22T23:43:50,198 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:50,198 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,203 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:50,203 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319090203 2024-11-22T23:43:50,203 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319150203 2024-11-22T23:43:50,203 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:50,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN}, {pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN}, {pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN}, {pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN}] 2024-11-22T23:43:50,203 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:50,206 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,208 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,209 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:50,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,211 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:50,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:50,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:50,211 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:50,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:50,211 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f9788bd5240@7054b052 rejected from java.util.concurrent.ThreadPoolExecutor@7a2d5708[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:50,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN because future has completed 2024-11-22T23:43:50,213 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f9788bd5240@dae36c2 rejected from java.util.concurrent.ThreadPoolExecutor@7a2d5708[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:50,217 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,217 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,320 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,320 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,345 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T23:43:50,345 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T23:43:50,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:43:50,347 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T23:43:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T23:43:50,348 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T23:43:50,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution 2024-11-22T23:43:50,349 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution Metrics about Tables on a single HBase RegionServer 2024-11-22T23:43:50,423 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,423 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,453 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,453 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:50,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,453 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:50,454 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:50,454 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:50,454 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:50,454 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:50,454 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:50,455 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:50,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:50,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:50,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:50,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,461 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,461 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,461 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:50,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,462 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:50,462 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:50,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc to restore RIT 2024-11-22T23:43:50,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46 to restore RIT 2024-11-22T23:43:50,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:50,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:50,466 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319060466 2024-11-22T23:43:50,466 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:50,466 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,466 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,466 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,466 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,471 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:50,471 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319090471 2024-11-22T23:43:50,471 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319150471 2024-11-22T23:43:50,471 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T23:43:50,471 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN because future has completed 2024-11-22T23:43:50,476 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:50,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:50,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:50,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN because future has completed 2024-11-22T23:43:50,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:50,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:50,480 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:50,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:50,480 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:50,480 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,480 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:50,526 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,526 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,628 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,628 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,629 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(122): Close b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:50,629 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:50,630 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1722): Closing b75b8feae225dafecff6deed65f9c1cc, disabling compactions & flushes 2024-11-22T23:43:50,630 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:50,630 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:50,630 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. after waiting 0 ms 2024-11-22T23:43:50,630 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:50,636 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:43:50,637 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc. 2024-11-22T23:43:50,637 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] regionserver.HRegion(1676): Region close journal for b75b8feae225dafecff6deed65f9c1cc: Waiting for close lock at 1732319030629Running coprocessor pre-close hooks at 1732319030630 (+1 ms)Disabling compacts and flushes for region at 1732319030630Disabling writes for close at 1732319030630Writing region close event to WAL at 1732319030630Running coprocessor post-close hooks at 1732319030636 (+6 ms)Closed at 1732319030637 (+1 ms) 2024-11-22T23:43:50,639 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=37}] handler.UnassignRegionHandler(157): Closed b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:50,721 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,721 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:50,721 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,721 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:50,721 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:50,722 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:50,722 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:50,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:50,722 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:50,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6] 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,725 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:50,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:50,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:50,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:50,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,729 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,729 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T23:43:50,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,730 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:50,731 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:50,731 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:50,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc to restore RIT 2024-11-22T23:43:50,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46 to restore RIT 2024-11-22T23:43:50,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:50,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:50,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319060733 2024-11-22T23:43:50,734 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:50,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,738 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:50,738 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319090738 2024-11-22T23:43:50,738 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319150738 2024-11-22T23:43:50,738 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:50,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED 2024-11-22T23:43:50,741 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:50,742 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:50,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN because future has completed 2024-11-22T23:43:50,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:50,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=33 2024-11-22T23:43:50,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 in 268 msec 2024-11-22T23:43:50,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:50,747 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,747 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=33, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:50,835 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,835 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,937 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,937 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:50,989 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:50,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:50,989 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:50,989 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:50,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:50,989 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:50,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:50,989 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:50,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5] 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,992 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:50,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:50,993 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:50,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:50,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:50,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,994 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,995 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:50,995 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,996 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,996 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:50,996 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:50,996 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:50,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:50,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:50,997 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:50,997 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:50,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:50,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:50,999 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc to restore RIT 2024-11-22T23:43:51,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46 to restore RIT 2024-11-22T23:43:51,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:51,000 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:51,000 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319061000 2024-11-22T23:43:51,000 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:51,000 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:51,001 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,001 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,001 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,005 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:51,005 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319091005 2024-11-22T23:43:51,005 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319151005 2024-11-22T23:43:51,005 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:51,005 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:51,006 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:51,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN in 803 msec 2024-11-22T23:43:51,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN because future has completed 2024-11-22T23:43:51,011 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:51,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:51,011 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:51,011 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,011 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:51,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:51,012 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f9788bd5240@6f1ce08 rejected from java.util.concurrent.ThreadPoolExecutor@1d3f9d5d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:51,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:51,013 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f9788bd5240@1a710f0c rejected from java.util.concurrent.ThreadPoolExecutor@1d3f9d5d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:51,040 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,040 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,142 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,142 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,255 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:51,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,255 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:51,255 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:51,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:51,256 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:51,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:51,256 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:51,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:51,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:51,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:51,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:51,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,262 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,262 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,262 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,262 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:51,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,263 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:51,263 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:51,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:51,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:51,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,268 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46 to restore RIT 2024-11-22T23:43:51,268 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:51,268 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:51,268 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319061268 2024-11-22T23:43:51,268 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:51,269 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,269 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,269 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,273 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:51,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319091273 2024-11-22T23:43:51,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319151273 2024-11-22T23:43:51,273 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:51,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,274 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,274 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN because future has completed 2024-11-22T23:43:51,276 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:51,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:51,276 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:51,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:51,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:51,280 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:51,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:51,280 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:51,280 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,280 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:51,348 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,429 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:51,430 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:51,430 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 287e1377df0169cd54d478142b508d46, disabling compactions & flushes 2024-11-22T23:43:51,430 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:51,430 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:51,430 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. after waiting 0 ms 2024-11-22T23:43:51,430 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:51,436 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:43:51,437 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46. 2024-11-22T23:43:51,437 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 287e1377df0169cd54d478142b508d46: Waiting for close lock at 1732319031430Running coprocessor pre-close hooks at 1732319031430Disabling compacts and flushes for region at 1732319031430Disabling writes for close at 1732319031430Writing region close event to WAL at 1732319031430Running coprocessor post-close hooks at 1732319031437 (+7 ms)Closed at 1732319031437 2024-11-22T23:43:51,440 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 287e1377df0169cd54d478142b508d46 2024-11-22T23:43:51,450 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,523 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 9 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,523 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:51,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,523 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:51,523 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:51,523 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:51,523 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:51,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:51,524 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:51,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:51,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:51,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:51,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:51,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:51,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,531 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:51,531 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:51,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:51,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:51,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46 to restore RIT 2024-11-22T23:43:51,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:51,534 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:51,534 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319061534 2024-11-22T23:43:51,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:51,535 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,535 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,535 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,539 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:51,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319091539 2024-11-22T23:43:51,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319151539 2024-11-22T23:43:51,539 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:51,540 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=34 updating hbase:meta row=287e1377df0169cd54d478142b508d46, regionState=CLOSED 2024-11-22T23:43:51,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=34, state=RUNNABLE, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:51,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:51,544 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:51,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:51,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=34 2024-11-22T23:43:51,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 in 268 msec 2024-11-22T23:43:51,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:51,548 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=34, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:51,553 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,553 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,656 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,656 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,758 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,758 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,789 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 10 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,789 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:51,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,789 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:51,789 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:51,789 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:51,789 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:51,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:51,790 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:51,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:51,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:51,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:51,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:51,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:51,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:51,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:51,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:51,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:51,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:51,797 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:51,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:51,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:51,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:51,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46 to restore RIT 2024-11-22T23:43:51,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:51,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:51,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319061801 2024-11-22T23:43:51,801 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:51,801 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,801 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,801 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:51,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:51,806 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:51,806 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319091806 2024-11-22T23:43:51,806 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319151806 2024-11-22T23:43:51,806 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-22T23:43:51,806 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:51,808 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:51,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN in 1.6050 sec 2024-11-22T23:43:51,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:51,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:51,814 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:51,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:51,814 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:51,814 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:51,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:51,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:51,814 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:51,815 ERROR [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$841/0x00007f9788bd5240@3812ec88 rejected from java.util.concurrent.ThreadPoolExecutor@63a6b17c[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 1] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:51,861 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:51,964 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:52,056 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 11 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,056 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:52,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,057 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:52,057 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:52,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:52,057 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:52,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:52,057 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:52,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:52,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:52,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:52,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:52,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:52,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:52,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:52,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:52,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,066 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:52,066 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:52,066 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:52,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,068 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:52,069 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:52,070 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319062070 2024-11-22T23:43:52,070 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:52,070 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,070 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,073 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:52,073 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319092073 2024-11-22T23:43:52,074 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319152074 2024-11-22T23:43:52,074 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:52,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,075 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN because future has completed 2024-11-22T23:43:52,077 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:52,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:52,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:52,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:52,082 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:52,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:52,082 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:52,082 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,082 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:52,231 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(122): Close 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:52,231 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:52,232 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1722): Closing 6ea6628446fa75243da49391aac996c2, disabling compactions & flushes 2024-11-22T23:43:52,232 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:52,232 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:52,232 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. after waiting 0 ms 2024-11-22T23:43:52,232 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:52,238 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:43:52,239 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2. 2024-11-22T23:43:52,239 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] regionserver.HRegion(1676): Region close journal for 6ea6628446fa75243da49391aac996c2: Waiting for close lock at 1732319032232Running coprocessor pre-close hooks at 1732319032232Disabling compacts and flushes for region at 1732319032232Disabling writes for close at 1732319032232Writing region close event to WAL at 1732319032232Running coprocessor post-close hooks at 1732319032239 (+7 ms)Closed at 1732319032239 2024-11-22T23:43:52,242 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=39}] handler.UnassignRegionHandler(157): Closed 6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:52,324 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 12 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,324 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:52,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,324 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:52,324 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:52,324 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:52,324 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:52,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:52,325 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:52,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:52,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13] 2024-11-22T23:43:52,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14] 2024-11-22T23:43:52,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:52,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:52,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:52,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:52,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:52,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,330 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:52,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:52,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,331 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:52,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,332 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:52,332 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:52,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:52,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:52,336 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319062336 2024-11-22T23:43:52,337 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:52,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,341 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:52,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319092341 2024-11-22T23:43:52,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319152341 2024-11-22T23:43:52,341 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:52,343 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6ea6628446fa75243da49391aac996c2, regionState=CLOSED 2024-11-22T23:43:52,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:52,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:52,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:52,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=35 2024-11-22T23:43:52,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 in 269 msec 2024-11-22T23:43:52,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:52,349 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=35, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:52,375 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:52,477 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:52,580 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:52,591 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 13 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:52,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,591 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:52,591 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:52,592 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:52,592 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:52,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:52,592 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:52,593 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:52,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:52,594 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:52,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:52,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:52,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:52,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,598 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:52,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,599 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:52,599 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:52,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2 to restore RIT 2024-11-22T23:43:52,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:52,615 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319062615 2024-11-22T23:43:52,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:52,616 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,616 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,620 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:52,620 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319092620 2024-11-22T23:43:52,620 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319152620 2024-11-22T23:43:52,620 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-22T23:43:52,620 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,621 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:52,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN in 2.4180 sec 2024-11-22T23:43:52,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:52,626 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:52,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:52,626 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:52,626 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,626 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:52,870 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 14 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,870 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:52,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:52,870 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:52,870 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:52,870 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:52,870 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:52,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:52,871 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:52,872 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:52,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:52,874 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-22T23:43:52,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:52,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:52,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:52,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:52,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:52,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,876 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:52,877 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:52,877 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:52,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,878 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:52,878 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:52,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:52,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:52,880 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:52,881 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319062881 2024-11-22T23:43:52,881 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:52,881 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:52,884 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,884 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,884 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:52,885 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:52,885 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319092885 2024-11-22T23:43:52,885 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319152885 2024-11-22T23:43:52,885 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:52,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSING, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:52,886 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41373 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,43005,1732319020630, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:43:52,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN because future has completed 2024-11-22T23:43:52,888 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:52,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630}] 2024-11-22T23:43:52,889 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:53,042 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:53,042 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:53,042 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 1bfcae741ae1bdf7847ea9d7f176cbb4, disabling compactions & flushes 2024-11-22T23:43:53,042 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:53,042 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:53,042 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. after waiting 0 ms 2024-11-22T23:43:53,042 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:53,048 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:43:53,049 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4. 2024-11-22T23:43:53,049 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 1bfcae741ae1bdf7847ea9d7f176cbb4: Waiting for close lock at 1732319033042Running coprocessor pre-close hooks at 1732319033042Disabling compacts and flushes for region at 1732319033042Disabling writes for close at 1732319033042Writing region close event to WAL at 1732319033043 (+1 ms)Running coprocessor post-close hooks at 1732319033049 (+6 ms)Closed at 1732319033049 2024-11-22T23:43:53,051 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:53,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED 2024-11-22T23:43:53,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:53,056 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:53,056 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,056 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=40, ppid=36, state=SUCCESS, hasLock=true; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:53,135 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 15 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,135 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:53,135 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,135 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:53,135 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:53,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:53,136 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:53,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:53,136 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:53,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:53,138 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:53,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17] 2024-11-22T23:43:53,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18] 2024-11-22T23:43:53,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:53,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:53,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:53,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:53,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,141 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:53,142 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,142 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:53,142 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:43:53,143 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,143 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:53,143 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:53,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,145 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,145 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:53,146 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319063146 2024-11-22T23:43:53,146 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:53,146 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,149 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:53,149 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319093149 2024-11-22T23:43:53,149 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319153149 2024-11-22T23:43:53,149 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:53,150 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=36 updating hbase:meta row=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, regionLocation=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=36, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 because future has completed 2024-11-22T23:43:53,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:53,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=36 2024-11-22T23:43:53,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 in 266 msec 2024-11-22T23:43:53,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:53,157 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=36, ppid=32, state=SUCCESS, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:53,400 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 16 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:53,400 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,400 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:53,400 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:53,400 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:53,400 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:53,400 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:53,400 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:53,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:53,402 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:53,402 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:53,402 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:53,402 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:53,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:53,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:53,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=WAITING:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T23:43:53,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,407 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:53,407 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:53,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,411 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN to state=OFFLINE, location=null, table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4 to restore RIT 2024-11-22T23:43:53,411 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319063411 2024-11-22T23:43:53,411 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:53,411 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,415 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:53,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319093415 2024-11-22T23:43:53,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319153415 2024-11-22T23:43:53,415 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:53,415 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=36, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:53,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=32 2024-11-22T23:43:53,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN in 3.2130 sec 2024-11-22T23:43:53,419 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:53,419 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,419 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=32, ppid=31, state=SUCCESS, hasLock=true; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:53,665 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 17 exec state=pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:53,665 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,665 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:53,665 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:53,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:53,665 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:53,665 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:53,666 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:53,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T23:43:53,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:53,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:53,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:53,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:53,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=31, state=WAITING:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T23:43:53,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,672 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:53,672 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:53,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,676 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319063676 2024-11-22T23:43:53,676 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:53,676 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_CONFIRM, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,679 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:53,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319093679 2024-11-22T23:43:53,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319153680 2024-11-22T23:43:53,680 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:53,680 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:53,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T23:43:53,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution in 3.7440 sec 2024-11-22T23:43:53,684 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:53,684 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,684 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:53,930 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 18 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,930 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:53,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,930 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:53,930 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:53,930 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:53,930 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:53,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:53,930 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:53,931 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T23:43:53,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T23:43:53,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:53,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:53,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:53,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:53,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,935 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:53,936 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,937 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,938 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,938 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:53,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-22T23:43:53,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,939 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:53,939 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:53,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:53,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:53,942 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319063942 2024-11-22T23:43:53,942 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:53,942 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_ADD_REPLICATION_BARRIER, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:53,946 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:53,946 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319093946 2024-11-22T23:43:53,946 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319153946 2024-11-22T23:43:53,946 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:53,946 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:53,948 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319033947"}]},"ts":"1732319033947"} 2024-11-22T23:43:53,950 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta 2024-11-22T23:43:53,950 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED 2024-11-22T23:43:53,950 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:53,950 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:53,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:53,950 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:54,176 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:43:54,196 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 19 exec state=pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,196 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:54,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:54,196 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:54,196 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:54,197 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:54,197 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:54,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:54,197 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:54,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T23:43:54,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:54,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:54,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:54,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:54,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:54,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:54,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:54,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,202 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T23:43:54,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,203 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:54,203 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:54,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:54,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:54,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:54,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:54,206 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319064206 2024-11-22T23:43:54,206 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:54,206 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_SET_DISABLED_TABLE_STATE, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,210 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:54,210 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319094210 2024-11-22T23:43:54,210 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319154210 2024-11-22T23:43:54,210 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:54,211 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319034210"}]},"ts":"1732319034210"} 2024-11-22T23:43:54,213 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution, state=DISABLED in hbase:meta 2024-11-22T23:43:54,213 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set testRecoveryAndDoubleExecution to state=DISABLED 2024-11-22T23:43:54,213 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:54,214 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-22T23:43:54,214 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:54,215 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:54,460 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 20 exec state=pid=31, state=SUCCESS, hasLock=true; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,460 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-22T23:43:54,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:54,461 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:54,461 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:54,461 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-22T23:43:54,461 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-22T23:43:54,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-22T23:43:54,461 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:43:54,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=31, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.DisableTableProcedure) stack ids=[0, 1, 2, 3, 22, 23] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=32, ppid=31, class=org.apache.hadoop.hbase.master.procedure.CloseTableRegionsProcedure) stack ids=[4, 21] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=33, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[5, 8] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=37, ppid=33, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[6, 7] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=34, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[9, 12] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=38, ppid=34, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[10, 11] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=35, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[13, 16] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=39, ppid=35, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[14, 15] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=36, ppid=32, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[17, 20] 2024-11-22T23:43:54,463 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=40, ppid=36, class=org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure) stack ids=[18, 19] 2024-11-22T23:43:54,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:43:54,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTable 2024-11-22T23:43:54,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=7, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTable 2024-11-22T23:43:54,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=12, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTable 2024-11-22T23:43:54,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:54,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=16, state=SUCCESS, hasLock=false; DisableTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:54,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=21, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testDisableTableMultipleTimes 2024-11-22T23:43:54,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution held the lock before restarting, call acquireLock to restore it. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=testRecoveryAndDoubleExecution didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=b75b8feae225dafecff6deed65f9c1cc, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=34, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=287e1377df0169cd54d478142b508d46, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=35, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=6ea6628446fa75243da49391aac996c2, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=36, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution, region=1bfcae741ae1bdf7847ea9d7f176cbb4, UNASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=37, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b75b8feae225dafecff6deed65f9c1cc, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=38, ppid=34, state=SUCCESS, hasLock=false; CloseRegionProcedure 287e1377df0169cd54d478142b508d46, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=39, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea6628446fa75243da49391aac996c2, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=40, ppid=36, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bfcae741ae1bdf7847ea9d7f176cbb4, server=a511e45b3024,43005,1732319020630 didn't hold the lock before restarting, skip acquiring lock. 2024-11-22T23:43:54,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-22T23:43:54,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,467 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=a511e45b3024,43005,1732319020630, table=hbase:meta, region=1588230740 2024-11-22T23:43:54,467 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,43005,1732319020630, state=OPEN 2024-11-22T23:43:54,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:54,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:43:54,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:54,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:43:54,471 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732319064471 2024-11-22T23:43:54,472 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:43:54,472 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=31, state=RUNNABLE:DISABLE_TABLE_POST_OPERATION, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b75b8feae225dafecff6deed65f9c1cc, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=287e1377df0169cd54d478142b508d46, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea6628446fa75243da49391aac996c2, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1bfcae741ae1bdf7847ea9d7f176cbb4, regionState=CLOSED, lastHost=a511e45b3024,43005,1732319020630, regionLocation=a511e45b3024,43005,1732319020630, openSeqNum=2 2024-11-22T23:43:54,475 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:43:54,475 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732319094475 2024-11-22T23:43:54,475 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732319154475 2024-11-22T23:43:54,475 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-22T23:43:54,475 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-22T23:43:54,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=testRecoveryAndDoubleExecution in 5.6190 sec 2024-11-22T23:43:54,725 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-22T23:43:54,727 INFO [Time-limited test {}] procedure.TestTableDDLProcedureBase(67): Tear down, remove table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable testRecoveryAndDoubleExecution 2024-11-22T23:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] ipc.CallRunner(138): callId: 12 service: MasterService methodName: DisableTable size: 91 connection: 172.17.0.2:52690 deadline: 1732319094728, exception=org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution 2024-11-22T23:43:54,729 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: DISABLE, Table Name: default:testRecoveryAndDoubleExecution failed with org.apache.hadoop.hbase.TableNotEnabledException: testRecoveryAndDoubleExecution at org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure.preflightChecks(AbstractStateMachineTableProcedure.java:163) at org.apache.hadoop.hbase.master.procedure.DisableTableProcedure.(DisableTableProcedure.java:78) at org.apache.hadoop.hbase.master.HMaster$13.run(HMaster.java:2831) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.disableTable(HMaster.java:2815) at org.apache.hadoop.hbase.master.MasterRpcServices.disableTable(MasterRpcServices.java:874) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:54,729 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(1553): Table: testRecoveryAndDoubleExecution already disabled, so just deleting it. 2024-11-22T23:43:54,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete testRecoveryAndDoubleExecution 2024-11-22T23:43:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,732 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=42, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,733 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=42, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:43:54,741 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:54,741 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46 2024-11-22T23:43:54,741 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:54,741 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:54,745 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/f1, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/f2, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/recovered.edits] 2024-11-22T23:43:54,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/f1, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/f2, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/recovered.edits] 2024-11-22T23:43:54,745 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/f1, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/f2, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/recovered.edits] 2024-11-22T23:43:54,745 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/f1, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/f2, FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/recovered.edits] 2024-11-22T23:43:54,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/recovered.edits/4.seqid to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/archive/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc/recovered.edits/4.seqid 2024-11-22T23:43:54,756 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/recovered.edits/4.seqid to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/archive/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46/recovered.edits/4.seqid 2024-11-22T23:43:54,756 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/recovered.edits/4.seqid to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/archive/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4/recovered.edits/4.seqid 2024-11-22T23:43:54,756 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/b75b8feae225dafecff6deed65f9c1cc 2024-11-22T23:43:54,757 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/recovered.edits/4.seqid to hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/archive/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2/recovered.edits/4.seqid 2024-11-22T23:43:54,757 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/287e1377df0169cd54d478142b508d46 2024-11-22T23:43:54,757 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/1bfcae741ae1bdf7847ea9d7f176cbb4 2024-11-22T23:43:54,758 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/default/testRecoveryAndDoubleExecution/6ea6628446fa75243da49391aac996c2 2024-11-22T23:43:54,758 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived testRecoveryAndDoubleExecution regions 2024-11-22T23:43:54,761 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=42, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,765 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 4 rows of testRecoveryAndDoubleExecution from hbase:meta 2024-11-22T23:43:54,768 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'testRecoveryAndDoubleExecution' descriptor. 2024-11-22T23:43:54,769 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=42, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,769 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'testRecoveryAndDoubleExecution' from region states. 2024-11-22T23:43:54,769 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319034769"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:54,769 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319034769"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:54,769 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319034769"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:54,770 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319034769"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:54,772 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 4 regions from META 2024-11-22T23:43:54,772 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b75b8feae225dafecff6deed65f9c1cc, NAME => 'testRecoveryAndDoubleExecution,,1732319027103.b75b8feae225dafecff6deed65f9c1cc.', STARTKEY => '', ENDKEY => 'a'}, {ENCODED => 287e1377df0169cd54d478142b508d46, NAME => 'testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46.', STARTKEY => 'a', ENDKEY => 'b'}, {ENCODED => 6ea6628446fa75243da49391aac996c2, NAME => 'testRecoveryAndDoubleExecution,b,1732319027103.6ea6628446fa75243da49391aac996c2.', STARTKEY => 'b', ENDKEY => 'c'}, {ENCODED => 1bfcae741ae1bdf7847ea9d7f176cbb4, NAME => 'testRecoveryAndDoubleExecution,c,1732319027103.1bfcae741ae1bdf7847ea9d7f176cbb4.', STARTKEY => 'c', ENDKEY => ''}] 2024-11-22T23:43:54,772 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'testRecoveryAndDoubleExecution' as deleted. 2024-11-22T23:43:54,773 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"testRecoveryAndDoubleExecution","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732319034772"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:54,774 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table testRecoveryAndDoubleExecution state from META 2024-11-22T23:43:54,775 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=42, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=testRecoveryAndDoubleExecution 2024-11-22T23:43:54,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; DeleteTableProcedure table=testRecoveryAndDoubleExecution in 45 msec 2024-11-22T23:43:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:43:54,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for testRecoveryAndDoubleExecution 2024-11-22T23:43:54,845 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:testRecoveryAndDoubleExecution completed 2024-11-22T23:43:54,853 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestDisableTableProcedure#testRecoveryAndDoubleExecution Thread=219 (was 222), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=8915 (was 8935) 2024-11-22T23:43:54,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T23:43:54,854 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:43:54,854 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:54,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:54,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:54,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:54,861 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T23:43:54,861 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1950020881, stopped=false 2024-11-22T23:43:54,861 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a511e45b3024,41373,1732319019939 2024-11-22T23:43:54,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:43:54,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:43:54,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:54,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:54,863 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:54,863 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:43:54,863 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestTableDDLProcedureBase.cleanupTest(TestTableDDLProcedureBase.java:52) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:54,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:54,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:43:54,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:43:54,864 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,43005,1732319020630' ***** 2024-11-22T23:43:54,864 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:43:54,864 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:43:54,864 INFO [RS:0;a511e45b3024:43005 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:43:54,864 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,43005,1732319020630 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a511e45b3024:43005. 2024-11-22T23:43:54,865 DEBUG [RS:0;a511e45b3024:43005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:54,865 DEBUG [RS:0;a511e45b3024:43005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:43:54,865 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:43:54,866 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T23:43:54,866 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T23:43:54,866 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:43:54,866 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:54,866 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:43:54,866 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:43:54,866 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:43:54,866 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:43:54,870 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=25.04 KB heapSize=40.99 KB 2024-11-22T23:43:54,907 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:43:54,933 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/info/c505f75c873d48f8a906c3a39a668b4c is 108, key is testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46./info:/1732319034769/DeleteFamily/seqid=0 2024-11-22T23:43:54,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741844_1020 (size=6669) 2024-11-22T23:43:55,066 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:55,267 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:55,347 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.21 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/info/c505f75c873d48f8a906c3a39a668b4c 2024-11-22T23:43:55,407 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/ns/b3501ad3caac4d7c916151cd394688b0 is 106, key is testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46./ns:/1732319034761/DeleteFamily/seqid=0 2024-11-22T23:43:55,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741845_1021 (size=6009) 2024-11-22T23:43:55,467 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:55,597 INFO [regionserver/a511e45b3024:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T23:43:55,597 INFO [regionserver/a511e45b3024:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T23:43:55,667 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:55,813 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/ns/b3501ad3caac4d7c916151cd394688b0 2024-11-22T23:43:55,839 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/rep_barrier/e405f90204b64ce3913c4dfdc2db3d7e is 115, key is testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46./rep_barrier:/1732319034761/DeleteFamily/seqid=0 2024-11-22T23:43:55,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741846_1022 (size=6121) 2024-11-22T23:43:55,867 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:56,068 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:56,245 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=648 B at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/rep_barrier/e405f90204b64ce3913c4dfdc2db3d7e 2024-11-22T23:43:56,268 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:56,271 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/table/f009d387a17949a6b688adab8f817523 is 109, key is testRecoveryAndDoubleExecution,a,1732319027103.287e1377df0169cd54d478142b508d46./table:/1732319034761/DeleteFamily/seqid=0 2024-11-22T23:43:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741847_1023 (size=6158) 2024-11-22T23:43:56,468 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:56,668 DEBUG [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:56,677 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.55 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/table/f009d387a17949a6b688adab8f817523 2024-11-22T23:43:56,690 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/info/c505f75c873d48f8a906c3a39a668b4c as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/info/c505f75c873d48f8a906c3a39a668b4c 2024-11-22T23:43:56,701 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/info/c505f75c873d48f8a906c3a39a668b4c, entries=12, sequenceid=77, filesize=6.5 K 2024-11-22T23:43:56,704 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/ns/b3501ad3caac4d7c916151cd394688b0 as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/ns/b3501ad3caac4d7c916151cd394688b0 2024-11-22T23:43:56,713 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/ns/b3501ad3caac4d7c916151cd394688b0, entries=8, sequenceid=77, filesize=5.9 K 2024-11-22T23:43:56,715 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/rep_barrier/e405f90204b64ce3913c4dfdc2db3d7e as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/rep_barrier/e405f90204b64ce3913c4dfdc2db3d7e 2024-11-22T23:43:56,725 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/rep_barrier/e405f90204b64ce3913c4dfdc2db3d7e, entries=6, sequenceid=77, filesize=6.0 K 2024-11-22T23:43:56,727 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/.tmp/table/f009d387a17949a6b688adab8f817523 as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/table/f009d387a17949a6b688adab8f817523 2024-11-22T23:43:56,742 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/table/f009d387a17949a6b688adab8f817523, entries=9, sequenceid=77, filesize=6.0 K 2024-11-22T23:43:56,749 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~25.04 KB/25645, heapSize ~40.93 KB/41912, currentSize=0 B/0 for 1588230740 in 1876ms, sequenceid=77, compaction requested=false 2024-11-22T23:43:56,759 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/data/hbase/meta/1588230740/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1 2024-11-22T23:43:56,760 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:43:56,760 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:43:56,760 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319034866Running coprocessor pre-close hooks at 1732319034866Disabling compacts and flushes for region at 1732319034866Disabling writes for close at 1732319034866Obtaining lock to block concurrent updates at 1732319034870 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1732319034870Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=25645, getHeapSize=41912, getOffHeapSize=0, getCellsCount=202 at 1732319034877 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732319034879 (+2 ms)Flushing 1588230740/info: creating writer at 1732319034881 (+2 ms)Flushing 1588230740/info: appending metadata at 1732319034923 (+42 ms)Flushing 1588230740/info: closing flushed file at 1732319034926 (+3 ms)Flushing 1588230740/ns: creating writer at 1732319035390 (+464 ms)Flushing 1588230740/ns: appending metadata at 1732319035406 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732319035406Flushing 1588230740/rep_barrier: creating writer at 1732319035821 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732319035838 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732319035838Flushing 1588230740/table: creating writer at 1732319036253 (+415 ms)Flushing 1588230740/table: appending metadata at 1732319036270 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732319036270Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fba0e53: reopening flushed file at 1732319036687 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54d6d99a: reopening flushed file at 1732319036703 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4917c79b: reopening flushed file at 1732319036714 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dff5c69: reopening flushed file at 1732319036725 (+11 ms)Finished flush of dataSize ~25.04 KB/25645, heapSize ~40.93 KB/41912, currentSize=0 B/0 for 1588230740 in 1876ms, sequenceid=77, compaction requested=false at 1732319036749 (+24 ms)Writing region close event to WAL at 1732319036753 (+4 ms)Running coprocessor post-close hooks at 1732319036760 (+7 ms)Closed at 1732319036760 2024-11-22T23:43:56,760 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:43:56,869 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,43005,1732319020630; all regions closed. 2024-11-22T23:43:56,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741834_1010 (size=30181) 2024-11-22T23:43:56,879 DEBUG [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/oldWALs 2024-11-22T23:43:56,879 INFO [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C43005%2C1732319020630.meta:.meta(num 1732319023660) 2024-11-22T23:43:56,901 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/WALs/a511e45b3024,43005,1732319020630/a511e45b3024%2C43005%2C1732319020630.1732319022754 not finished, retry = 0 2024-11-22T23:43:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741832_1008 (size=4131) 2024-11-22T23:43:57,005 DEBUG [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/oldWALs 2024-11-22T23:43:57,005 INFO [RS:0;a511e45b3024:43005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C43005%2C1732319020630:(num 1732319022754) 2024-11-22T23:43:57,005 DEBUG [RS:0;a511e45b3024:43005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:57,005 INFO [RS:0;a511e45b3024:43005 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:43:57,005 INFO [RS:0;a511e45b3024:43005 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:43:57,005 INFO [RS:0;a511e45b3024:43005 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T23:43:57,006 INFO [RS:0;a511e45b3024:43005 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:43:57,006 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:43:57,006 INFO [RS:0;a511e45b3024:43005 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43005 2024-11-22T23:43:57,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,43005,1732319020630 2024-11-22T23:43:57,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:43:57,010 INFO [RS:0;a511e45b3024:43005 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:43:57,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,43005,1732319020630] 2024-11-22T23:43:57,014 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,43005,1732319020630 already deleted, retry=false 2024-11-22T23:43:57,014 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a511e45b3024,43005,1732319020630 expired; onlineServers=0 2024-11-22T23:43:57,015 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a511e45b3024,41373,1732319019939' ***** 2024-11-22T23:43:57,015 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T23:43:57,015 INFO [M:0;a511e45b3024:41373 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:43:57,015 INFO [M:0;a511e45b3024:41373 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:43:57,015 DEBUG [M:0;a511e45b3024:41373 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T23:43:57,015 DEBUG [M:0;a511e45b3024:41373 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T23:43:57,015 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T23:43:57,015 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319022415 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732319022415,5,FailOnTimeoutGroup] 2024-11-22T23:43:57,015 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319022415 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732319022415,5,FailOnTimeoutGroup] 2024-11-22T23:43:57,016 INFO [M:0;a511e45b3024:41373 {}] hbase.ChoreService(370): Chore service for: master/a511e45b3024:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T23:43:57,016 INFO [M:0;a511e45b3024:41373 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:43:57,016 DEBUG [M:0;a511e45b3024:41373 {}] master.HMaster(1795): Stopping service threads 2024-11-22T23:43:57,016 INFO [M:0;a511e45b3024:41373 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T23:43:57,016 INFO [M:0;a511e45b3024:41373 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:57,016 ERROR [M:0;a511e45b3024:41373 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-3,5,PEWorkerGroup] Thread[HFileArchiver-4,5,PEWorkerGroup] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T23:43:57,017 INFO [M:0;a511e45b3024:41373 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:57,017 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T23:43:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T23:43:57,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:57,019 DEBUG [M:0;a511e45b3024:41373 {}] zookeeper.ZKUtil(347): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T23:43:57,019 WARN [M:0;a511e45b3024:41373 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T23:43:57,020 INFO [M:0;a511e45b3024:41373 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/.lastflushedseqids 2024-11-22T23:43:57,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741848_1024 (size=72) 2024-11-22T23:43:57,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:57,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10142106af00001, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:57,114 INFO [RS:0;a511e45b3024:43005 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:43:57,114 INFO [RS:0;a511e45b3024:43005 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,43005,1732319020630; zookeeper connection closed. 2024-11-22T23:43:57,114 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b68b268 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b68b268 2024-11-22T23:43:57,115 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T23:43:57,433 INFO [M:0;a511e45b3024:41373 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:57,433 INFO [M:0;a511e45b3024:41373 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T23:43:57,433 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:43:57,433 INFO [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:57,433 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:57,433 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:43:57,433 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:57,433 INFO [M:0;a511e45b3024:41373 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=149.09 KB heapSize=178.88 KB 2024-11-22T23:43:57,452 DEBUG [M:0;a511e45b3024:41373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e53dfb6b3f22454b912a6adb824777d4 is 82, key is hbase:meta,,1/info:regioninfo/1732319023745/Put/seqid=0 2024-11-22T23:43:57,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741849_1025 (size=5672) 2024-11-22T23:43:57,858 INFO [M:0;a511e45b3024:41373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e53dfb6b3f22454b912a6adb824777d4 2024-11-22T23:43:57,888 DEBUG [M:0;a511e45b3024:41373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bbef6dcf95e432794a77c2db3dad8dd is 1206, key is \x00\x00\x00\x00\x00\x00\x00\x16/proc:d/1732319028775/Put/seqid=0 2024-11-22T23:43:57,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741850_1026 (size=11517) 2024-11-22T23:43:58,296 INFO [M:0;a511e45b3024:41373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bbef6dcf95e432794a77c2db3dad8dd 2024-11-22T23:43:58,323 DEBUG [M:0;a511e45b3024:41373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cfda4a3d789a4ef0922a7c1e3de0d629 is 69, key is a511e45b3024,43005,1732319020630/rs:state/1732319022532/Put/seqid=0 2024-11-22T23:43:58,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741851_1027 (size=5156) 2024-11-22T23:43:58,730 INFO [M:0;a511e45b3024:41373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cfda4a3d789a4ef0922a7c1e3de0d629 2024-11-22T23:43:58,739 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e53dfb6b3f22454b912a6adb824777d4 as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e53dfb6b3f22454b912a6adb824777d4 2024-11-22T23:43:58,746 INFO [M:0;a511e45b3024:41373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e53dfb6b3f22454b912a6adb824777d4, entries=8, sequenceid=328, filesize=5.5 K 2024-11-22T23:43:58,747 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bbef6dcf95e432794a77c2db3dad8dd as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bbef6dcf95e432794a77c2db3dad8dd 2024-11-22T23:43:58,756 INFO [M:0;a511e45b3024:41373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bbef6dcf95e432794a77c2db3dad8dd, entries=39, sequenceid=328, filesize=11.2 K 2024-11-22T23:43:58,758 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cfda4a3d789a4ef0922a7c1e3de0d629 as hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cfda4a3d789a4ef0922a7c1e3de0d629 2024-11-22T23:43:58,766 INFO [M:0;a511e45b3024:41373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/79a4146b-d3c8-e4b2-8c6e-9047bfd6dcbc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cfda4a3d789a4ef0922a7c1e3de0d629, entries=1, sequenceid=328, filesize=5.0 K 2024-11-22T23:43:58,767 INFO [M:0;a511e45b3024:41373 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.09 KB/152670, heapSize ~178.58 KB/182864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1334ms, sequenceid=328, compaction requested=false 2024-11-22T23:43:58,769 INFO [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:58,769 DEBUG [M:0;a511e45b3024:41373 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319037433Disabling compacts and flushes for region at 1732319037433Disabling writes for close at 1732319037433Obtaining lock to block concurrent updates at 1732319037433Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732319037434 (+1 ms)Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=152670, getHeapSize=183104, getOffHeapSize=0, getCellsCount=389 at 1732319037434Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732319037435 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732319037435Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732319037451 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732319037451Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732319037866 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732319037887 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732319037887Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732319038304 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732319038322 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732319038322Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62d2829f: reopening flushed file at 1732319038738 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2155c299: reopening flushed file at 1732319038746 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57828679: reopening flushed file at 1732319038757 (+11 ms)Finished flush of dataSize ~149.09 KB/152670, heapSize ~178.58 KB/182864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1334ms, sequenceid=328, compaction requested=false at 1732319038767 (+10 ms)Writing region close event to WAL at 1732319038769 (+2 ms)Closed at 1732319038769 2024-11-22T23:43:58,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45113 is added to blk_1073741830_1006 (size=174653) 2024-11-22T23:43:58,780 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:43:58,780 INFO [M:0;a511e45b3024:41373 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T23:43:58,780 INFO [M:0;a511e45b3024:41373 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41373 2024-11-22T23:43:58,781 INFO [M:0;a511e45b3024:41373 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:43:58,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:58,883 INFO [M:0;a511e45b3024:41373 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:43:58,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41373-0x10142106af00000, quorum=127.0.0.1:55056, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:58,889 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@156ffeaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T23:43:58,892 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6fb08d37{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T23:43:58,892 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T23:43:58,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@419718a4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T23:43:58,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ba59cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/hadoop.log.dir/,STOPPED} 2024-11-22T23:43:58,897 WARN [BP-363626384-172.17.0.2-1732319017109 heartbeating to localhost/127.0.0.1:35361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T23:43:58,897 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T23:43:58,897 WARN [BP-363626384-172.17.0.2-1732319017109 heartbeating to localhost/127.0.0.1:35361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-363626384-172.17.0.2-1732319017109 (Datanode Uuid 0efe8e3a-43f8-4371-8c09-d1207ffbc6b7) service to localhost/127.0.0.1:35361 2024-11-22T23:43:58,897 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T23:43:58,898 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/data/data1/current/BP-363626384-172.17.0.2-1732319017109 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T23:43:58,899 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/cluster_55e52785-e335-af0c-96e4-28cfcfb2fc76/data/data2/current/BP-363626384-172.17.0.2-1732319017109 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T23:43:58,899 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T23:43:58,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27b19a13{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T23:43:58,912 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37fd905b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T23:43:58,912 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T23:43:58,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c6c3a78{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T23:43:58,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46ae9a80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6662d4ef-a6b8-9849-99f0-a8c1f864f485/hadoop.log.dir/,STOPPED} 2024-11-22T23:43:58,923 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T23:43:58,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down